diff --git a/.editorconfig b/.editorconfig index c5f3028c5036..d7fe7ad5e244 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,rs,py,sh}] indent_size = 4 -# .cirrus.yml, etc. +# ci.yml, etc. [*.yml] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index c9cf4a7d9cd0..25303e742a76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ src/clientversion.cpp export-subst +src/CMakeLists.txt export-subst diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 83922b54cbf4..7c894ab7df50 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -78,7 +78,7 @@ body: id: os attributes: label: Operating system and version - placeholder: e.g. "MacOS Ventura 13.2" or "Ubuntu 22.04 LTS" + placeholder: e.g. "MacOS 26.0" or "Ubuntu 26.04 LTS" validations: required: true - type: textarea @@ -90,4 +90,3 @@ body: e.g. OS/CPU and disk type, network connectivity validations: required: false - diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml deleted file mode 100644 index 2a486b3f2b43..000000000000 --- a/.github/ISSUE_TEMPLATE/good_first_issue.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Good First Issue -description: (Regular devs only) Suggest a new good first issue -labels: [good first issue] -body: - - type: markdown - attributes: - value: | - Please add the label "good first issue" manually before or after opening - - A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution - - Motivate the issue and explain the solution briefly - - type: textarea - id: motivation - attributes: - label: Motivation - description: Motivate the issue - validations: - required: true - - type: textarea - id: solution - attributes: - label: Possible solution - description: Describe a possible solution - validations: - required: false - - type: textarea - id: useful-skills - attributes: - label: Useful Skills - description: For example, “`std::thread`”, “Qt6 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. - value: | - * Compiling Bitcoin Core from source - * Running the C++ unit tests and the Python functional tests - * ... - - type: textarea - attributes: - label: Guidance for new contributors - description: Please leave this to automatically add the footer for new contributors - value: | - Want to work on this issue? - - For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. - diff --git a/.github/actions/restore-caches/action.yml b/.github/actions/cache/restore/action.yml similarity index 77% rename from .github/actions/restore-caches/action.yml rename to .github/actions/cache/restore/action.yml index 8dc35d4902ed..2cc5b53a1e1f 100644 --- a/.github/actions/restore-caches/action.yml +++ b/.github/actions/cache/restore/action.yml @@ -1,43 +1,51 @@ name: 'Restore Caches' description: 'Restore ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: - name: Restore Ccache cache id: ccache-cache - uses: cirruslabs/cache/restore@v4 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} restore-keys: | ccache-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore depends sources cache id: depends-sources - uses: cirruslabs/cache/restore@v4 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-sources-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore built depends cache id: depends-built - uses: cirruslabs/cache/restore@v4 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-built-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore previous releases cache id: previous-releases - uses: cirruslabs/cache/restore@v4 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} restore-keys: | previous-releases-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: export cache hits shell: bash diff --git a/.github/actions/cache/restore/internal/action.yml b/.github/actions/cache/restore/internal/action.yml new file mode 100644 index 000000000000..43dd206345df --- /dev/null +++ b/.github/actions/cache/restore/internal/action.yml @@ -0,0 +1,43 @@ +name: 'Cache Restore' +description: 'Restore a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to restore' + required: true + key: + description: 'An explicit key for restoring the cache' + required: true + restore-keys: + description: 'An ordered multiline string listing prefix-matched restore keys' + required: false + default: '' + provider: + description: 'The cache provider to use' + required: true +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.warp.outputs.cache-hit || steps.gha.outputs.cache-hit }} + cache-primary-key: + description: 'The primary key used to restore the cache' + value: ${{ steps.warp.outputs.cache-primary-key || steps.gha.outputs.cache-primary-key }} +runs: + using: 'composite' + steps: + - name: Restore cache with WarpBuild + id: warp + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/restore@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + - name: Restore cache with GitHub Actions + id: gha + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} diff --git a/.github/actions/save-caches/action.yml b/.github/actions/cache/save/action.yml similarity index 79% rename from .github/actions/save-caches/action.yml rename to .github/actions/cache/save/action.yml index 0e3b31246c61..5c543b3f9e78 100644 --- a/.github/actions/save-caches/action.yml +++ b/.github/actions/cache/save/action.yml @@ -1,5 +1,9 @@ name: 'Save Caches' description: 'Save ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: @@ -11,29 +15,33 @@ runs: echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}" - name: Save Ccache cache - uses: cirruslabs/cache/save@v4 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }} with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} + provider: ${{ inputs.provider }} - name: Save depends sources cache - uses: cirruslabs/cache/save@v4 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }} with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save built depends cache - uses: cirruslabs/cache/save@v4 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}} with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save previous releases cache - uses: cirruslabs/cache/save@v4 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}} with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} + provider: ${{ inputs.provider }} diff --git a/.github/actions/cache/save/internal/action.yml b/.github/actions/cache/save/internal/action.yml new file mode 100644 index 000000000000..12f4156a072f --- /dev/null +++ b/.github/actions/cache/save/internal/action.yml @@ -0,0 +1,28 @@ +name: 'Cache Save' +description: 'Save a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache' + required: true + key: + description: 'An explicit key for saving the cache' + required: true + provider: + description: 'The cache provider to use' + required: true +runs: + using: 'composite' + steps: + - name: Save cache with WarpBuild + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/save@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + + - name: Save cache with GitHub Actions + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/save@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} diff --git a/.github/actions/clear-files/action.yml b/.github/actions/clear-files/action.yml new file mode 100644 index 000000000000..0008f825ff0d --- /dev/null +++ b/.github/actions/clear-files/action.yml @@ -0,0 +1,12 @@ +name: 'Clear unnecessary files' +description: 'Clear out unnecessary files to make space on the VM' +runs: + using: 'composite' + steps: + - name: Clear unnecessary files + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + run: | + set +o errexit + sudo bash -c '(ionice -c 3 nice -n 19 rm -rf /usr/share/dotnet/ /usr/local/graalvm/ /usr/local/.ghcup/ /usr/local/share/powershell /usr/local/share/chromium /usr/local/lib/android /usr/local/lib/node_modules)&' diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index c78df86b6cf1..074b47ce348b 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -1,49 +1,53 @@ name: 'Configure Docker' description: 'Set up Docker build driver and configure build cache args' inputs: - use-cirrus: - description: 'Use cirrus cache' - required: true + provider: + description: 'The cache provider to use' + required: false + default: 'gha' runs: using: 'composite' steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx for Warp cache + if: ${{ inputs.provider == 'warp' }} + uses: docker/setup-buildx-action@v4 with: - # Use host network to allow access to cirrus gha cache running on the host driver-opts: | network=host - # This is required to allow buildkit to access the actions cache + - name: Set up Docker Buildx + if: ${{ inputs.provider != 'warp' }} + uses: docker/setup-buildx-action@v4 + + # This is required when using the gha cache backend with a manual docker buildx invocation. + # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN + # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - name: Expose actions cache variables - uses: actions/github-script@v6 + uses: actions/github-script@v8 with: script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + Object.keys(process.env).forEach(function (key) { + if (key.startsWith('ACTIONS_')) { + core.info(`Exporting ${key}`); + core.exportVariable(key, process.env[key]); + } + }); - name: Construct docker build cache args shell: bash run: | - # Configure docker build cache backend - # - # On forks the gha cache will work but will use Github's cache backend. - # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN - # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - - # Use cirrus cache host - if [[ ${{ inputs.use-cirrus }} == 'true' ]]; then - url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}" - else - url_args="" + cache_options="scope=${CONTAINER_NAME}" + if [[ "${{ inputs.provider }}" == "warp" ]]; then + cache_options="url=http://127.0.0.1:49160/,version=1,${cache_options}" fi + # Configure docker build cache backend # Always optimistically --cache‑from in case a cache blob exists - args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") + args=(--cache-from "type=gha,${cache_options}") - # If this is a push to the default branch, also add --cache‑to to save the cache + # Only add --cache-to when pushing to the default branch. if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then - args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") + args+=(--cache-to "type=gha,mode=max,ignore-error=true,${cache_options}") fi # Always `--load` into docker images (needed when using the `docker-container` build driver). diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index aae5016bdce9..e2a26b7184d1 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -17,7 +17,7 @@ runs: - name: Set cache hashes shell: bash run: | - echo "DEPENDS_HASH=$(git ls-tree HEAD depends "ci/test/$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + echo "DEPENDS_HASH=$(git ls-tree HEAD depends "$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV echo "PREVIOUS_RELEASES_HASH=$(git ls-tree HEAD test/get_previous_releases.py | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - name: Get container name diff --git a/.github/ci-test-each-commit-exec.py b/.github/ci-test-each-commit-exec.py index 3b2eaeeb5964..402e73527be1 100755 --- a/.github/ci-test-each-commit-exec.py +++ b/.github/ci-test-each-commit-exec.py @@ -10,10 +10,11 @@ def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) try: - return subprocess.run(cmd, check=True, **kwargs) + return subprocess.run(cmd, **kwargs) except Exception as e: - sys.exit(e) + sys.exit(str(e)) def main(): @@ -21,11 +22,12 @@ def main(): run(["git", "log", "-1"]) num_procs = int(run(["nproc"], stdout=subprocess.PIPE).stdout) + build_dir = "ci_build" run([ "cmake", "-B", - "build", + build_dir, "-Werror=dev", # Use clang++, because it is a bit faster and uses less memory than g++ "-DCMAKE_C_COMPILER=clang", @@ -36,29 +38,31 @@ def main(): "-DAPPEND_CXXFLAGS='-O3 -g2'", "-DAPPEND_CFLAGS='-O3 -g2'", "-DCMAKE_BUILD_TYPE=Debug", - "-DWERROR=ON", - "-DWITH_ZMQ=ON", - "-DBUILD_GUI=ON", - "-DBUILD_BENCH=ON", - "-DBUILD_FUZZ_BINARY=ON", - "-DWITH_USDT=ON", - "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function", + "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", + "--preset=dev-mode", + # Tolerate unused (member) functions in intermediate commits in a pull request + "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function -Wno-error=unused-function", ]) - run(["cmake", "--build", "build", "-j", str(num_procs)]) + + if run(["cmake", "--build", build_dir, "-j", str(num_procs)], check=False).returncode != 0: + print("Build failure. Verbose build follows.") + run(["cmake", "--build", build_dir, "-j1", "--verbose"]) + run([ "ctest", "--output-on-failure", "--stop-on-failure", "--test-dir", - "build", + build_dir, "-j", str(num_procs), ]) run([ sys.executable, - "./build/test/functional/test_runner.py", + f"./{build_dir}/test/functional/test_runner.py", "-j", str(num_procs * 2), + "--failfast", "--combinedlogslen=99999999", ]) diff --git a/.github/ci-windows-cross.py b/.github/ci-windows-cross.py new file mode 100755 index 000000000000..bf13f81ac75c --- /dev/null +++ b/.github/ci-windows-cross.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import argparse +import os +import shlex +import subprocess +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent.parent / "test")) +from download_utils import download_script_assets + + +def run(cmd, **kwargs): + print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) + try: + return subprocess.run(cmd, **kwargs) + except Exception as e: + sys.exit(str(e)) + + +def print_version(): + bitcoind = Path.cwd() / "bin" / "bitcoind.exe" + run([str(bitcoind), "-version"]) + + +def check_manifests(): + release_dir = Path.cwd() / "bin" + manifest_path = release_dir / "bitcoind.manifest" + + cmd_bitcoind_manifest = [ + "mt.exe", + "-nologo", + f"-inputresource:{release_dir / 'bitcoind.exe'}", + f"-out:{manifest_path}", + ] + run(cmd_bitcoind_manifest) + print(manifest_path.read_text()) + + skipped = { # Skip as they currently do not have manifests + "fuzz.exe", + "bench_bitcoin.exe", + } + for entry in release_dir.iterdir(): + if entry.suffix.lower() != ".exe": + continue + if entry.name in skipped: + print(f"Skipping {entry.name} (no manifest present)") + continue + print(f"Checking {entry.name}") + run(["mt.exe", "-nologo", f"-inputresource:{entry}", "-validate_manifest"]) + + +def prepare_tests(): + workspace = Path.cwd() + config_path = workspace / "test" / "config.ini" + rpcauth_path = workspace / "share" / "rpcauth" / "rpcauth.py" + replacements = { + "SRCDIR=": f"SRCDIR={workspace}", + "BUILDDIR=": f"BUILDDIR={workspace}", + "RPCAUTH=": f"RPCAUTH={rpcauth_path}", + } + lines = config_path.read_text().splitlines() + for index, line in enumerate(lines): + for prefix, new_value in replacements.items(): + if line.startswith(prefix): + lines[index] = new_value + break + content = "\n".join(lines) + "\n" + config_path.write_text(content) + print(content) + previous_releases_dir = Path(os.environ["PREVIOUS_RELEASES_DIR"]) + cmd_download_prev_rel = [ + sys.executable, + str(workspace / "test" / "get_previous_releases.py"), + "--target-dir", + str(previous_releases_dir), + ] + run(cmd_download_prev_rel) + run([sys.executable, "-m", "pip", "install", "pyzmq"]) + + dest = workspace / "unit_test_data" + download_script_assets(dest) + + +def run_functional_tests(): + workspace = Path.cwd() + num_procs = str(os.process_cpu_count()) + test_runner_cmd = [ + sys.executable, + str(workspace / "test" / "functional" / "test_runner.py"), + "--jobs", + num_procs, + "--quiet", + f"--tmpdirprefix={workspace / '_ _'}", + "--combinedlogslen=99999999", + *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), + # feature_unsupported_utxo_db.py fails on Windows because of emojis in the test data directory. + "--exclude", + "feature_unsupported_utxo_db.py", + ] + run(test_runner_cmd) + + # Run feature_unsupported_utxo_db sequentially in ASCII-only tmp dir, + # because it is excluded above due to lack of UTF-8 support in the + # ancient release. + cmd_feature_unsupported_db = [ + sys.executable, + str(workspace / "test" / "functional" / "feature_unsupported_utxo_db.py"), + "--previous-releases", + "--tmpdir", + str(Path(workspace) / "test_feature_unsupported_utxo_db"), + ] + run(cmd_feature_unsupported_db) + + +def run_unit_tests(): + workspace = Path.cwd() + os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data") + # Can't use ctest here like other jobs as we don't have a CMake build tree. + commands = [ + ["./bin/test_bitcoin-qt.exe"], + # Intentionally run sequentially here, to catch test case failures caused by dirty global state from prior test cases: + ["./bin/test_bitcoin.exe", "-l", "test_suite"], + ["./src/secp256k1/bin/exhaustive_tests.exe"], + ["./src/secp256k1/bin/noverify_tests.exe"], + ["./src/secp256k1/bin/tests.exe"], + ["./src/univalue/object.exe"], + ["./src/univalue/unitester.exe"], + ] + for cmd in commands: + run(cmd) + + +def main(): + parser = argparse.ArgumentParser(description="Utility to run Windows CI steps.") + steps = list(map(lambda f: f.__name__, [ + print_version, + check_manifests, + prepare_tests, + run_unit_tests, + run_functional_tests, + ])) + parser.add_argument("step", choices=steps, help="CI step to perform.") + args = parser.parse_args() + + os.environ.setdefault( + "PREVIOUS_RELEASES_DIR", + str(Path.cwd() / "previous_releases"), + ) + + exec(f'{args.step}()') + + +if __name__ == "__main__": + main() diff --git a/.github/ci-windows.py b/.github/ci-windows.py new file mode 100755 index 000000000000..2635682266f5 --- /dev/null +++ b/.github/ci-windows.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import argparse +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent.parent / "test")) +from download_utils import download_script_assets + + +def run(cmd, **kwargs): + print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) + try: + return subprocess.run(cmd, **kwargs) + except Exception as e: + sys.exit(str(e)) + + +GENERATE_OPTIONS = { + "standard": [ + "-DBUILD_BENCH=ON", + "-DBUILD_KERNEL_LIB=ON", + "-DBUILD_UTIL_CHAINSTATE=ON", + "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", + ], + "fuzz": [ + "-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON", + "-DVCPKG_MANIFEST_FEATURES=wallet", + "-DBUILD_FOR_FUZZING=ON", + "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", + ], +} + + +def github_import_vs_env(_ci_type): + vswhere_path = Path(os.environ["ProgramFiles(x86)"]) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + installation_path = run( + [str(vswhere_path), "-latest", "-property", "installationPath"], + capture_output=True, + text=True, + ).stdout.strip() + vsdevcmd = Path(installation_path) / "Common7" / "Tools" / "vsdevcmd.bat" + comspec = os.environ["COMSPEC"] + output = run( + f'"{comspec}" /s /c ""{vsdevcmd}" -arch=x64 -no_logo && set"', + capture_output=True, + text=True, + ).stdout + github_env = os.environ["GITHUB_ENV"] + with open(github_env, "a") as env_file: + for line in output.splitlines(): + if "=" not in line: + continue + name, value = line.split("=", 1) + env_file.write(f"{name}={value}\n") + + +def generate(ci_type): + command = [ + "cmake", + "-B", + "build", + "-Werror=dev", + "--preset=vs2026", + # Using x64-windows-release for both host and target triplets + # to ensure vcpkg builds only release packages, thereby optimizing + # build time. + # See https://github.com/microsoft/vcpkg/issues/50927. + "-DVCPKG_HOST_TRIPLET=x64-windows-release", + "-DVCPKG_TARGET_TRIPLET=x64-windows-release", + ] + GENERATE_OPTIONS[ci_type] + if run(command, check=False).returncode != 0: + print("=== ⚠️ ===") + print("Generate failure! Network issue? Retry once ...") + time.sleep(12) + print("=== ⚠️ ===") + run(command) + + +def build(_ci_type): + command = [ + "cmake", + "--build", + "build", + "--config", + "Release", + ] + if run(command + ["-j", str(os.process_cpu_count())], check=False).returncode != 0: + print("Build failure. Verbose build follows.") + run(command + ["-j1", "--verbose"]) + + +def check_manifests(ci_type): + if ci_type != "standard": + print(f"Skipping manifest validation for '{ci_type}' ci type.") + return + + release_dir = Path.cwd() / "build" / "bin" / "Release" + manifest_path = release_dir / "bitcoind.manifest" + cmd_bitcoind_manifest = [ + "mt.exe", + "-nologo", + f"-inputresource:{release_dir / 'bitcoind.exe'}", + f"-out:{manifest_path}", + ] + run(cmd_bitcoind_manifest) + print(manifest_path.read_text()) + + skips = { # Skip as they currently do not have manifests + "fuzz.exe", + "bench_bitcoin.exe", + "test_bitcoin-qt.exe", + "bitcoin-chainstate.exe", + } + for entry in release_dir.iterdir(): + if entry.suffix.lower() != ".exe": + continue + if entry.name in skips: + print(f"Skipping {entry.name} (no manifest present)") + continue + print(f"Checking {entry.name}") + cmd_check_manifest = [ + "mt.exe", + "-nologo", + f"-inputresource:{entry}", + "-validate_manifest", + ] + run(cmd_check_manifest) + + +def prepare_tests(ci_type): + workspace = Path.cwd() + if ci_type == "standard": + run([sys.executable, "-m", "pip", "install", "pyzmq"]) + dest = workspace / "unit_test_data" + download_script_assets(dest) + elif ci_type == "fuzz": + repo_dir = str(workspace / "qa-assets") + clone_cmd = [ + "git", + "clone", + "--depth=1", + "https://github.com/bitcoin-core/qa-assets", + repo_dir, + ] + run(clone_cmd) + print("Using qa-assets repo from commit ...") + run(["git", "-C", repo_dir, "log", "-1"]) + + +def run_tests(ci_type): + workspace = Path.cwd() + build_dir = workspace / "build" + num_procs = str(os.process_cpu_count()) + release_bin = build_dir / "bin" / "Release" + + if ci_type == "standard": + os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data") + test_envs = { + "BITCOIN_BIN": "bitcoin.exe", + "BITCOIND": "bitcoind.exe", + "BITCOINCLI": "bitcoin-cli.exe", + "BITCOIN_BENCH": "bench_bitcoin.exe", + "BITCOINTX": "bitcoin-tx.exe", + "BITCOINUTIL": "bitcoin-util.exe", + "BITCOINWALLET": "bitcoin-wallet.exe", + "BITCOINCHAINSTATE": "bitcoin-chainstate.exe", + } + for var, exe in test_envs.items(): + os.environ[var] = str(release_bin / exe) + + ctest_cmd = [ + "ctest", + "--test-dir", + str(build_dir), + "--output-on-failure", + "--stop-on-failure", + "-j", + num_procs, + "--build-config", + "Release", + ] + run(ctest_cmd) + + test_cmd = [ + sys.executable, + str(build_dir / "test" / "functional" / "test_runner.py"), + "--jobs", + num_procs, + "--quiet", + f"--tmpdirprefix={workspace / '_ _'}", + "--combinedlogslen=99999999", + *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), + ] + run(test_cmd) + + elif ci_type == "fuzz": + os.environ["BITCOINFUZZ"] = str(release_bin / "fuzz.exe") + fuzz_cmd = [ + sys.executable, + str(build_dir / "test" / "fuzz" / "test_runner.py"), + "--par", + num_procs, + "--loglevel", + "DEBUG", + str(workspace / "qa-assets" / "fuzz_corpora"), + ] + run(fuzz_cmd) + + +def main(): + parser = argparse.ArgumentParser(description="Utility to run Windows CI steps.") + parser.add_argument("ci_type", choices=GENERATE_OPTIONS, help="CI type to run.") + steps = list(map(lambda f: f.__name__, [ + github_import_vs_env, + generate, + build, + check_manifests, + prepare_tests, + run_tests, + ])) + parser.add_argument("step", choices=steps, help="CI step to perform.") + args = parser.parse_args() + + exec(f'{args.step}("{args.ci_type}")') + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52d21ef3ab5d..3f039997d62b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,6 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type. - REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners defaults: run: @@ -29,57 +27,61 @@ defaults: shell: bash jobs: - runners: - name: 'determine runners' - runs-on: ubuntu-latest - outputs: - use-cirrus-runners: ${{ steps.runners.outputs.use-cirrus-runners }} - steps: - - id: runners - run: | - if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then - echo "use-cirrus-runners=true" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using Cirrus Runners" - else - echo "use-cirrus-runners=false" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using GitHub-hosted runners" - fi - test-each-commit: - name: 'test each commit' - runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 - timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. + name: 'test ancestor commits' + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} env: - MAX_COUNT: 6 + TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. + if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 + timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. steps: - name: Determine fetch depth run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" - - uses: actions/checkout@v5 + - &ANNOTATION_PR_NUMBER + name: Annotate with pull request number + # This annotation is machine-readable and can be used to assign a check + # run to its corresponding pull request. Running in all check runs is + # required, because check re-runs discard the annotations of other + # tasks in the test suite. + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "::notice title=debug_pull_request_number_str::${{ github.event.number }}" + fi + - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: ${{ env.FETCH_DEPTH }} - name: Determine commit range run: | - # Checkout HEAD~ and find the test base commit - # Checkout HEAD~ because it would be wasteful to rerun tests on the PR - # head commit that are already run by other jobs. + # Checkout HEAD~ and find the test base commit. + # Checkout HEAD~ because it would be wasteful to rerun + # tests on the PR head commit that are already run + # by other jobs. git checkout HEAD~ - # Figure out test base commit by listing ancestors of HEAD, excluding - # ancestors of the most recent merge commit, limiting the list to the - # newest MAX_COUNT ancestors, ordering it from oldest to newest, and - # taking the first one. + # Moreover, pull requests that contain a merge commit + # are generally draft pull requests that merge in other + # pull requests, so only check the relevant commits + # after the last merge commit. A merge commit could + # also be a subtree merge commit, which may be + # worthwhile to check. However, it is rare that the + # subtree merge commit is not the top commit (which + # would be skipped anyway by this task, because it is + # run by all other tasks). Also, `git rebase --exec` + # does not work on merge commits, so if this was + # important to check, the logic would have to be + # rewritten. # - # If the branch contains up to MAX_COUNT ancestor commits after the - # most recent merge commit, all of those commits will be tested. If it - # contains more, only the most recent MAX_COUNT commits will be - # tested. + # Figure out test base commit by listing ancestors of + # HEAD, excluding ancestors of the most recent merge + # commit, ordering them from oldest to newest, and + # taking the first one. # - # In the command below, the ^@ suffix is used to refer to all parents - # of the merge commit as described in: + # In the command below, the ^@ suffix is used to refer + # to all parents of the merge commit as described in: # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations - # and the ^ prefix is used to exclude these parents and all their - # ancestors from the rev-list output as described in: + # and the ^ prefix is used to exclude these parents + # and all their ancestors from the rev-list output + # as described in: # https://git-scm.com/docs/git-rev-list MERGE_BASE=$(git rev-list -n1 --merges HEAD) EXCLUDE_MERGE_BASE_ANCESTORS= @@ -87,14 +89,14 @@ jobs: if test -n "$MERGE_BASE"; then EXCLUDE_MERGE_BASE_ANCESTORS=^${MERGE_BASE}^@ fi - echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" + echo "TEST_BASE=$(git rev-list -n${{ github.event.pull_request.commits }} --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" - run: | git fetch origin "${GITHUB_BASE_REF}" git config user.email "ci@example.com" git config user.name "CI" - run: | sudo apt-get update - sudo apt-get install clang mold ccache build-essential cmake ninja-build pkgconf python3-zmq libevent-dev libboost-dev libsqlite3-dev systemtap-sdt-dev libzmq3-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev capnproto libcapnp-dev -y + sudo apt-get install clang mold ccache build-essential cmake ninja-build pkgconf python3-zmq libboost-dev libsqlite3-dev systemtap-sdt-dev libzmq3-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev capnproto libcapnp-dev -y sudo pip3 install --break-system-packages pycapnp - name: Compile and run tests run: | @@ -105,7 +107,7 @@ jobs: name: ${{ matrix.job-name }} # Use any image to support the xcode-select below, but hardcode version to avoid silent upgrades (and breaks). # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-14 + runs-on: macos-15 # When a contributor maintains a fork of the repo, any pull request they make # to their own fork, or to the main repository, will trigger two CI runs: @@ -123,30 +125,32 @@ jobs: include: - job-type: standard file-env: './ci/test/00_setup_env_mac_native.sh' - job-name: 'macOS 14 native, arm64, no depends, sqlite only, gui' + job-name: 'macOS native' - job-type: fuzz file-env: './ci/test/00_setup_env_mac_native_fuzz.sh' - job-name: 'macOS 14 native, arm64, fuzz' + job-name: 'macOS native, fuzz' env: DANGER_RUN_CI_ON_HOST: 1 - BASE_ROOT_DIR: ${{ github.workspace }} + BASE_ROOT_DIR: ${{ github.workspace }}/repo_archive steps: + - *ANNOTATION_PR_NUMBER + - &CHECKOUT name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: # Ensure the latest merged pull request state is used, even on re-runs. ref: &CHECKOUT_REF_TMPL ${{ github.event_name == 'pull_request' && github.ref || '' }} - name: Clang version run: | - # Use the earliest Xcode supported by the version of macOS denoted in + # Use the latest Xcode supported by the version of macOS denoted in # doc/release-notes-empty-template.md and providing at least the # minimum clang version denoted in doc/dependencies.md. - # See: https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes - sudo xcode-select --switch /Applications/Xcode_15.0.app + # See: https://developer.apple.com/documentation/xcode-release-notes/xcode-16_2-release-notes + sudo xcode-select --switch /Applications/Xcode_16.2.app clang --version - name: Install Homebrew packages @@ -155,40 +159,43 @@ jobs: run: | # A workaround for "The `brew link` step did not complete successfully" error. brew install --quiet python@3 || brew link --overwrite python@3 - brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@6 qrencode capnp - - - name: Install Python packages - run: | - git clone -b v2.1.0 https://github.com/capnproto/pycapnp - pip3 install ./pycapnp -C force-bundled-libcapnp=True --break-system-packages + brew install --quiet coreutils ninja pkgconf ccache boost zeromq qt@6 qrencode capnp - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" - name: Restore Ccache cache id: ccache-cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v5 with: path: ${{ env.CCACHE_DIR }} key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} restore-keys: ${{ github.job }}-${{ matrix.job-type }}-ccache- + - name: Create git archive + run: | + git log -1 + git archive --format=tar --prefix=repo_archive/ --output=repo.tar HEAD + tar -xf repo.tar + - name: CI script - run: ./ci/test_run_all.sh + run: | + cd repo_archive + ./ci/test_run_all.sh env: FILE_ENV: ${{ matrix.file-env }} - name: Save Ccache cache - uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.ccache-cache.outputs.cache-hit != 'true' with: path: ${{ env.CCACHE_DIR }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} + key: ${{ steps.ccache-cache.outputs.cache-primary-key }} windows-native-dll: name: ${{ matrix.job-name }} - runs-on: windows-2022 + runs-on: windows-2025-vs2026 if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} @@ -202,227 +209,218 @@ jobs: job-type: [standard, fuzz] include: - job-type: standard - generate-options: '-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DWERROR=ON' - job-name: 'Windows native, VS 2022' + job-name: 'Windows native, VS' - job-type: fuzz - generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="wallet" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON' - job-name: 'Windows native, fuzz, VS 2022' + job-name: 'Windows native, fuzz, VS' steps: + - *ANNOTATION_PR_NUMBER + - *CHECKOUT - - name: Configure Developer Command Prompt for Microsoft Visual C++ - # Using microsoft/setup-msbuild is not enough. - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: x64 + - &IMPORT_VS_ENV + name: Import Visual Studio env vars + run: py -3 .github/ci-windows.py "standard" github_import_vs_env - name: Get tool information - shell: pwsh run: | - cmake -version | Tee-Object -FilePath "cmake_version" - Write-Output "---" - msbuild -version | Tee-Object -FilePath "msbuild_version" - $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" + set -o errexit -o pipefail -o xtrace -o nounset + + cmake -version | tee cmake_version + echo '---' + msbuild.exe -version | tee msbuild_version + echo "${VCToolsVersion-}" | tee toolset_version py -3 --version - Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" bash --version - - name: Using vcpkg with MSBuild - run: | - echo "set(VCPKG_BUILD_TYPE release)" >> "${VCPKG_INSTALLATION_ROOT}/triplets/x64-windows.cmake" - # Workaround for libevent, which requires CMake 3.1 but is incompatible with CMake >= 4.0. - sed -i '1s/^/set(ENV{CMAKE_POLICY_VERSION_MINIMUM} 3.5)\n/' "${VCPKG_INSTALLATION_ROOT}/scripts/ports.cmake" - - - name: vcpkg tools cache - uses: actions/cache@v4 + - name: Restore vcpkg downloads cache + id: vcpkg-downloads-cache + uses: actions/cache/restore@v5 with: - path: C:/vcpkg/downloads/tools - key: ${{ github.job }}-vcpkg-tools + path: | + ~/AppData/Local/vcpkg/downloads/* + !~/AppData/Local/vcpkg/downloads/tools + key: ${{ github.job }}-vcpkg-downloads-${{ hashFiles('vcpkg.json') }} - name: Restore vcpkg binary cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@v5 id: vcpkg-binary-cache with: path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} + key: ${{ github.job }}-vcpkg-binary-release-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} - name: Generate build system run: | - cmake -B build -Werror=dev --preset vs2022 -DCMAKE_TOOLCHAIN_FILE="${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" ${{ matrix.generate-options }} + py -3 .github/ci-windows.py ${{ matrix.job-type }} generate - name: Save vcpkg binary cache - uses: actions/cache/save@v4 - if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' + uses: actions/cache/save@v5 + if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' with: path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} + key: ${{ steps.vcpkg-binary-cache.outputs.cache-primary-key }} + + - name: Save vcpkg downloads cache + uses: actions/cache/save@v5 + # Only save cache from the 'standard' job, as it includes the necessary downloads for other jobs in the matrix. If the matrix is modified, this may need amending. + if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && steps.vcpkg-downloads-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' + with: + # Cache the tools once as archives, but not redundantly in extracted form. + path: | + ~/AppData/Local/vcpkg/downloads/* + !~/AppData/Local/vcpkg/downloads/tools + key: ${{ steps.vcpkg-downloads-cache.outputs.cache-primary-key }} - name: Build - working-directory: build run: | - cmake --build . -j $NUMBER_OF_PROCESSORS --config Release + py -3 .github/ci-windows.py ${{ matrix.job-type }} build - - name: Get bitcoind manifest - if: matrix.job-type == 'standard' - working-directory: build + - name: Check executable manifests run: | - mt.exe -nologo -inputresource:bin/Release/bitcoind.exe -out:bitcoind.manifest - cat bitcoind.manifest - echo - mt.exe -nologo -inputresource:bin/Release/bitcoind.exe -validate_manifest - - - name: Run test suite - if: matrix.job-type == 'standard' - working-directory: build - env: - QT_PLUGIN_PATH: '${{ github.workspace }}\build\vcpkg_installed\x64-windows\Qt6\plugins' - run: | - ctest --output-on-failure --stop-on-failure -j $NUMBER_OF_PROCESSORS -C Release + py -3 .github/ci-windows.py ${{ matrix.job-type }} check_manifests - - name: Run functional tests - if: matrix.job-type == 'standard' - working-directory: build - env: - BITCOIN_BIN: '${{ github.workspace }}\build\bin\Release\bitcoin.exe' - BITCOIND: '${{ github.workspace }}\build\bin\Release\bitcoind.exe' - BITCOINCLI: '${{ github.workspace }}\build\bin\Release\bitcoin-cli.exe' - BITCOINTX: '${{ github.workspace }}\build\bin\Release\bitcoin-tx.exe' - BITCOINUTIL: '${{ github.workspace }}\build\bin\Release\bitcoin-util.exe' - BITCOINWALLET: '${{ github.workspace }}\build\bin\Release\bitcoin-wallet.exe' - TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} - run: py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix="${RUNNER_TEMP}" --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} - - - name: Clone corpora - if: matrix.job-type == 'fuzz' + - name: Prepare tests run: | - git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${RUNNER_TEMP}/qa-assets" - cd "${RUNNER_TEMP}/qa-assets" - echo "Using qa-assets repo from commit ..." - git log -1 + py -3 .github/ci-windows.py ${{ matrix.job-type }} prepare_tests - - name: Run fuzz tests - if: matrix.job-type == 'fuzz' - working-directory: build + - name: Run tests env: - BITCOINFUZZ: '${{ github.workspace }}\build\bin\Release\fuzz.exe' + TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}" run: | - py -3 test/fuzz/test_runner.py --par $NUMBER_OF_PROCESSORS --loglevel DEBUG "${RUNNER_TEMP}/qa-assets/fuzz_corpora" + py -3 .github/ci-windows.py ${{ matrix.job-type }} run_tests + + record-frozen-commit: + # Record frozen commit, so that the native tests on cross-builds can run on + # the exact same commit id of the build. + name: '[meta] record frozen commit' + runs-on: ubuntu-slim + outputs: + commit: ${{ steps.record-commit.outputs.commit }} + steps: + - *ANNOTATION_PR_NUMBER + - *CHECKOUT + - name: Record commit + id: record-commit + run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" windows-cross: - name: 'Linux->Windows cross, no tests' - needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' || 'ubuntu-24.04' }} + name: 'Windows-cross to x86_64, ${{ matrix.crt }}' + needs: record-frozen-commit + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-4x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} + strategy: + fail-fast: false + matrix: + crt: [msvcrt, ucrt] + include: + - crt: msvcrt + file-env: './ci/test/00_setup_env_win64_msvcrt.sh' + artifact-name: 'x86_64-w64-mingw32-executables' + - crt: ucrt + file-env: './ci/test/00_setup_env_win64.sh' + artifact-name: 'x86_64-w64-mingw32ucrt-executables' + env: - FILE_ENV: './ci/test/00_setup_env_win64.sh' + FILE_ENV: ${{ matrix.file-env }} DANGER_CI_ON_HOST_FOLDERS: 1 steps: - - *CHECKOUT + - *ANNOTATION_PR_NUMBER + + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.record-frozen-commit.outputs.commit }} - name: Configure environment uses: ./.github/actions/configure-environment - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }} + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: CI script run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: Upload built executables - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: x86_64-w64-mingw32-executables-${{ github.run_id }} + name: ${{ matrix.artifact-name }}-${{ github.run_id }} path: | + ${{ env.BASE_BUILD_DIR }}/bin/*.dll ${{ env.BASE_BUILD_DIR }}/bin/*.exe ${{ env.BASE_BUILD_DIR }}/src/secp256k1/bin/*.exe ${{ env.BASE_BUILD_DIR }}/src/univalue/*.exe ${{ env.BASE_BUILD_DIR }}/test/config.ini windows-native-test: - name: 'Windows, test cross-built' + name: 'Windows, ${{ matrix.crt }}, test cross-built' runs-on: windows-2022 - needs: windows-cross + needs: [windows-cross, record-frozen-commit] + + strategy: + fail-fast: false + matrix: + crt: [msvcrt, ucrt] + include: + - crt: msvcrt + artifact-name: 'x86_64-w64-mingw32-executables' + - crt: ucrt + artifact-name: 'x86_64-w64-mingw32ucrt-executables' env: PYTHONUTF8: 1 TEST_RUNNER_TIMEOUT_FACTOR: 40 steps: - - *CHECKOUT + - *ANNOTATION_PR_NUMBER + + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.record-frozen-commit.outputs.commit }} - name: Download built executables - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: - name: x86_64-w64-mingw32-executables-${{ github.run_id }} + name: ${{ matrix.artifact-name }}-${{ github.run_id }} - name: Run bitcoind.exe - run: ./bin/bitcoind.exe -version + run: py -3 .github/ci-windows-cross.py print_version - - name: Find mt.exe tool - shell: pwsh - run: | - $sdk_dir = (Get-ItemProperty 'HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows Kits\Installed Roots' -Name KitsRoot10).KitsRoot10 - $sdk_latest = (Get-ChildItem "$sdk_dir\bin" -Directory | Where-Object { $_.Name -match '^\d+\.\d+\.\d+\.\d+$' } | Sort-Object Name -Descending | Select-Object -First 1).Name - "MT_EXE=${sdk_dir}bin\${sdk_latest}\x64\mt.exe" >> $env:GITHUB_ENV - - - name: Get bitcoind manifest - shell: pwsh - run: | - & $env:MT_EXE -nologo -inputresource:bin\bitcoind.exe -out:bitcoind.manifest - Get-Content bitcoind.manifest - & $env:MT_EXE -nologo -inputresource:bin\bitcoind.exe -validate_manifest + - *IMPORT_VS_ENV - - name: Run unit tests - # Can't use ctest here like other jobs as we don't have a CMake build tree. - run: | - ./bin/test_bitcoin.exe -l test_suite # Intentionally run sequentially here, to catch test case failures caused by dirty global state from prior test cases. - ./src/secp256k1/bin/exhaustive_tests.exe - ./src/secp256k1/bin/noverify_tests.exe - ./src/secp256k1/bin/tests.exe - ./src/univalue/object.exe - ./src/univalue/unitester.exe - - - name: Run benchmarks - run: ./bin/bench_bitcoin.exe -sanity-check - - - name: Adjust paths in test/config.ini - shell: pwsh - run: | - (Get-Content "test/config.ini") -replace '(?<=^SRCDIR=).*', '${{ github.workspace }}' -replace '(?<=^BUILDDIR=).*', '${{ github.workspace }}' -replace '(?<=^RPCAUTH=).*', '${{ github.workspace }}/share/rpcauth/rpcauth.py' | Set-Content "test/config.ini" - Get-Content "test/config.ini" + - name: Check executable manifests + run: py -3 .github/ci-windows-cross.py check_manifests - - name: Set previous release directory - run: | - echo "PREVIOUS_RELEASES_DIR=${{ runner.temp }}/previous_releases" >> "$GITHUB_ENV" + - name: Prepare Windows test environment + run: py -3 .github/ci-windows-cross.py prepare_tests - - name: Get previous releases - working-directory: test - run: ./get_previous_releases.py --target-dir $PREVIOUS_RELEASES_DIR + - name: Run unit tests + run: py -3 .github/ci-windows-cross.py run_unit_tests - name: Run functional tests env: - # TODO: Fix the excluded test and re-enable it. - # feature_unsupported_utxo_db.py fails on windows because of emojis in the test data directory - EXCLUDE: '--exclude wallet_multiwallet.py,feature_unsupported_utxo_db.py' - TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} - run: py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix="$RUNNER_TEMP" --combinedlogslen=99999999 --timeout-factor=$TEST_RUNNER_TIMEOUT_FACTOR $EXCLUDE $TEST_RUNNER_EXTRA + TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}" + run: | + py -3 .github/ci-windows-cross.py run_functional_tests ci-matrix: name: ${{ matrix.name }} - needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && matrix.cirrus-runner || matrix.fallback-runner }} + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && matrix.warp-runner || matrix.fallback-runner }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: ${{ matrix.timeout-minutes }} @@ -434,73 +432,112 @@ jobs: fail-fast: false matrix: include: - - name: '32 bit ARM, unit tests, no functional tests' - cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode + - name: 'iwyu' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_native_iwyu.sh' + + - name: '32 bit ARM' + warp-runner: 'ubuntu-24.04-arm' # Warp's Arm runners don't support 32-bit mode currently fallback-runner: 'ubuntu-24.04-arm' timeout-minutes: 120 file-env: './ci/test/00_setup_env_arm.sh' + provider: 'gha' - - name: 'ASan + LSan + UBSan + integer, no depends, USDT' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + - name: 'ASan + LSan + UBSan + integer' + warp-runner: 'warp-ubuntu-2404-x64-8x' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - - name: 'macOS-cross, gui, no tests' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + - name: 'macOS-cross to arm64' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' - - name: 'No wallet, libbitcoinkernel' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + - name: 'macOS-cross to x86_64' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 - file-env: './ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh' + file-env: './ci/test/00_setup_env_mac_cross_intel.sh' - - name: 'no IPC, i686, DEBUG' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + - name: 'FreeBSD Cross' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_freebsd_cross.sh' + + - name: 'OpenBSD Cross' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_openbsd_cross.sh' + + - name: 'No wallet' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_native_nowallet.sh' + + - name: 'i686, no IPC' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_i686_no_ipc.sh' - - name: 'fuzzer,address,undefined,integer, no depends' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + - name: 'fuzzer,address,undefined,integer' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - - name: 'previous releases, depends DEBUG' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + - name: 'previous releases' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - - name: 'CentOS, depends, gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + - name: 'Alpine (musl)' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 - file-env: './ci/test/00_setup_env_native_centos.sh' + file-env: './ci/test/00_setup_env_native_alpine_musl.sh' - name: 'tidy' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tidy.sh' - - name: 'TSan, depends, no gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + - name: 'TSan' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' - - name: 'MSan, depends' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + - name: 'MSan, fuzz' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 180 + file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' + + - name: 'MSan' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_msan.sh' + - name: 'riscv32 bare metal, static libbitcoin_consensus' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' + timeout-minutes: 120 + file-env: './ci/test/00_setup_env_riscv_bare_cross.sh' + steps: + - *ANNOTATION_PR_NUMBER + - *CHECKOUT - name: Configure environment @@ -508,12 +545,18 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ matrix.provider || (github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha') }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }} + provider: ${{ matrix.provider || (github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha') }} + + - name: Clear unnecessary files + if: ${{ github.repository != 'bitcoin/bitcoin' || matrix.provider == 'gha' }} # Only needed on GHA runners + uses: ./.github/actions/clear-files - name: Enable bpfcc script if: ${{ env.CONTAINER_NAME == 'ci_native_asan' }} @@ -522,7 +565,7 @@ jobs: run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh - name: Set mmap_rnd_bits - if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }} + if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' || env.CONTAINER_NAME == 'ci_native_fuzz_msan' }} # Prevents crashes due to high ASLR entropy run: sudo sysctl -w vm.mmap_rnd_bits=28 @@ -530,19 +573,22 @@ jobs: run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ matrix.provider || (github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha') }} lint: name: 'lint' - needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ github.repository == 'bitcoin/bitcoin' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: CONTAINER_NAME: "bitcoin-linter" steps: + - *ANNOTATION_PR_NUMBER + - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: ref: *CHECKOUT_REF_TMPL fetch-depth: 0 @@ -550,14 +596,9 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker with: - use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }} + provider: ${{ github.repository == 'bitcoin/bitcoin' && 'warp' || 'gha' }} - name: CI script run: | - set -o xtrace - docker buildx build -t "$CONTAINER_NAME" $DOCKER_BUILD_CACHE_ARG --file "./ci/lint_imagefile" . - CIRRUS_PR_FLAG="" - if [ "${{ github.event_name }}" = "pull_request" ]; then - CIRRUS_PR_FLAG="-e CIRRUS_PR=1" - fi - docker run --rm $CIRRUS_PR_FLAG -v "$(pwd)":/bitcoin "$CONTAINER_NAME" + git worktree add ../lint-worktree HEAD + ../lint-worktree/ci/lint.py diff --git a/.gitignore b/.gitignore index 3b22e358dcaf..b92988f6a3d1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ # Only ignore unexpected patches *.patch +!ci/test/*.patch !contrib/guix/patches/*.patch !depends/patches/**/*.patch diff --git a/.python-version b/.python-version index 1445aee866ce..c8cfe3959183 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.14 +3.10 diff --git a/.tx/config b/.tx/config index a7550d0884dc..41051254f32f 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[o:bitcoin:p:bitcoin:r:qt-translation-030x] +[o:bitcoin:p:bitcoin:r:qt-translation-031x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c10fb73fad9..f09cec1b0e13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,12 +27,12 @@ get_directory_property(precious_variables CACHE_VARIABLES) # Project / Package metadata #============================= set(CLIENT_NAME "Bitcoin Core") -set(CLIENT_VERSION_MAJOR 30) +set(CLIENT_VERSION_MAJOR 31) set(CLIENT_VERSION_MINOR 99) set(CLIENT_VERSION_BUILD 0) set(CLIENT_VERSION_RC 0) set(CLIENT_VERSION_IS_RELEASE "false") -set(COPYRIGHT_YEAR "2025") +set(COPYRIGHT_YEAR "2026") # During the enabling of the CXX and CXXOBJ languages, we modify # CMake's compiler/linker invocation strings by appending the content @@ -69,12 +69,18 @@ set(CLIENT_BUGREPORT "https://github.com/bitcoin/bitcoin/issues") #============================= # Language setup #============================= -if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_HOST_APPLE) +if(CMAKE_VERSION VERSION_LESS 4.2 AND CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_HOST_APPLE) # We do not use the install_name_tool when cross-compiling for macOS. + # However, CMake < 4.2 still searches for Apple's version of the tool, + # which causes an error during configuration. + # See: + # - https://gitlab.kitware.com/cmake/cmake/-/issues/27069 + # - https://gitlab.kitware.com/cmake/cmake/-/merge_requests/10955 # So disable this tool check in further enable_language() commands. - set(CMAKE_PLATFORM_HAS_INSTALLNAME FALSE) + set(CMAKE_INSTALL_NAME_TOOL "${CMAKE_COMMAND} -E true") endif() enable_language(CXX) + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -102,77 +108,33 @@ option(BUILD_CLI "Build bitcoin-cli executable." ON) option(BUILD_TESTS "Build test_bitcoin and other unit test executables." ON) option(BUILD_TX "Build bitcoin-tx executable." ${BUILD_TESTS}) option(BUILD_UTIL "Build bitcoin-util executable." ${BUILD_TESTS}) +cmake_dependent_option(BUILD_GUI_TESTS "Build test_bitcoin-qt executable." ON "BUILD_GUI;BUILD_TESTS" OFF) option(BUILD_UTIL_CHAINSTATE "Build experimental bitcoin-chainstate executable." OFF) option(BUILD_KERNEL_LIB "Build experimental bitcoinkernel library." ${BUILD_UTIL_CHAINSTATE}) +cmake_dependent_option(BUILD_KERNEL_TEST "Build tests for the experimental bitcoinkernel library." ON "BUILD_KERNEL_LIB" OFF) option(ENABLE_WALLET "Enable wallet." ON) -if(ENABLE_WALLET) - if(VCPKG_TARGET_TRIPLET) - # Use of the `unofficial::` namespace is a vcpkg package manager convention. - find_package(unofficial-sqlite3 CONFIG REQUIRED) - else() - find_package(SQLite3 3.7.17 REQUIRED) - endif() -endif() cmake_dependent_option(BUILD_WALLET_TOOL "Build bitcoin-wallet tool." ${BUILD_TESTS} "ENABLE_WALLET" OFF) -option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting executables." OFF) -option(WERROR "Treat compiler warnings as errors." OFF) -option(WITH_CCACHE "Attempt to use ccache for compiling." ON) - +option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) option(WITH_ZMQ "Enable ZMQ notifications." OFF) -if(WITH_ZMQ) - find_package(ZeroMQ 4.0.0 MODULE REQUIRED) -endif() - +cmake_dependent_option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables." ON "NOT WIN32" OFF) +cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF "ENABLE_IPC" OFF) +option(WITH_EMBEDDED_ASMAP "Embed default ASMap data." ON) option(WITH_USDT "Enable tracepoints for Userspace, Statically Defined Tracing." OFF) -if(WITH_USDT) - find_package(USDT MODULE REQUIRED) -endif() - -option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) cmake_dependent_option(WITH_QRENCODE "Enable QR code support." ON "BUILD_GUI" OFF) -if(WITH_QRENCODE) - find_package(QRencode MODULE REQUIRED) - set(USE_QRCODE TRUE) -endif() - cmake_dependent_option(WITH_DBUS "Enable DBus support." ON "NOT CMAKE_SYSTEM_NAME MATCHES \"(Windows|Darwin)\" AND BUILD_GUI" OFF) -cmake_dependent_option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables." ON "NOT WIN32" OFF) -cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF "ENABLE_IPC" OFF) -if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS) - find_package(Libmultiprocess REQUIRED COMPONENTS Lib) - find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin - NAMES Libmultiprocess - ) -endif() - -cmake_dependent_option(BUILD_GUI_TESTS "Build test_bitcoin-qt executable." ON "BUILD_GUI;BUILD_TESTS" OFF) -if(BUILD_GUI) - set(qt_components Core Gui Widgets LinguistTools) - if(ENABLE_WALLET) - list(APPEND qt_components Network) - endif() - if(WITH_DBUS) - list(APPEND qt_components DBus) - set(USE_DBUS TRUE) - endif() - if(BUILD_GUI_TESTS) - list(APPEND qt_components Test) - endif() - find_package(Qt 6.2 MODULE REQUIRED - COMPONENTS ${qt_components} - ) - unset(qt_components) -endif() - option(BUILD_BENCH "Build bench_bitcoin executable." OFF) option(BUILD_FUZZ_BINARY "Build fuzz binary." OFF) option(BUILD_FOR_FUZZING "Build for fuzzing. Enabling this will disable all other targets and override BUILD_FUZZ_BINARY." OFF) +option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting executables." OFF) +option(CMAKE_COMPILE_WARNING_AS_ERROR "Treat compiler warnings as errors." OFF) +option(WITH_CCACHE "Attempt to use ccache for compiling." ON) + option(INSTALL_MAN "Install man pages." ON) set(APPEND_CPPFLAGS "" CACHE STRING "Preprocessor flags that are appended to the command line after all other flags added by the build system. This variable is intended for debugging and special builds.") @@ -185,11 +147,13 @@ string(APPEND CMAKE_CXX_COMPILE_OBJECT " ${APPEND_CPPFLAGS} ${APPEND_CXXFLAGS}") string(APPEND CMAKE_CXX_CREATE_SHARED_LIBRARY " ${APPEND_LDFLAGS}") string(APPEND CMAKE_CXX_LINK_EXECUTABLE " ${APPEND_LDFLAGS}") -set(configure_warnings) - -include(CheckLinkerSupportsPIE) -check_linker_supports_pie(configure_warnings) +# Check required C++ toolchain features early. +include(CheckCXXFeatures) +check_cxx_features() +#============================= +# Core interface library +#============================= # The core_interface library aims to encapsulate common build flags. # It is a usage requirement for all targets except for secp256k1, which # gets its flags by other means. @@ -201,6 +165,15 @@ target_link_libraries(core_interface INTERFACE $<$:core_interface_debug> ) +#============================= +# Option interaction +#============================= +# Compute the final state of build options. +# This must be done after all options have been initially set, +# as some options (e.g., BUILD_FOR_FUZZING) may override or adjust others. +# It must also come before any dependency checks to ensure that the set +# of dependencies checked is consistent with the final option values, +# that is, that all required, and only the required, dependencies are tested. if(BUILD_FOR_FUZZING) message(WARNING "BUILD_FOR_FUZZING=ON will disable all other targets and force BUILD_FUZZ_BINARY=ON.") set(BUILD_BITCOIN_BIN OFF) @@ -210,10 +183,12 @@ if(BUILD_FOR_FUZZING) set(BUILD_UTIL OFF) set(BUILD_UTIL_CHAINSTATE OFF) set(BUILD_KERNEL_LIB OFF) + set(BUILD_KERNEL_TEST OFF) set(BUILD_WALLET_TOOL OFF) set(BUILD_GUI OFF) set(ENABLE_EXTERNAL_SIGNER OFF) set(WITH_ZMQ OFF) + set(WITH_EMBEDDED_ASMAP OFF) set(BUILD_TESTS OFF) set(BUILD_GUI_TESTS OFF) set(BUILD_BENCH OFF) @@ -225,6 +200,83 @@ if(BUILD_FOR_FUZZING) ) endif() +#============================= +# Check and set PIC/PIE +#============================= +set(configure_warnings) +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") +include(CheckLinkerSupportsPIE) +# Set CMAKE_POSITION_INDEPENDENT_CODE early so it applies to all +# subsequent checks, including dependency packages and tool flags. +check_linker_supports_pie(configure_warnings) +endif() + +#============================= +# Find dependencies +#============================= +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +target_link_libraries(core_interface INTERFACE + Threads::Threads +) +endif() + +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") +include(AddBoostIfNeeded) +add_boost_if_needed() +endif() + +if(ENABLE_WALLET) + if(VCPKG_TARGET_TRIPLET) + # Use of the `unofficial::` namespace is a vcpkg package manager convention. + find_package(unofficial-sqlite3 CONFIG REQUIRED) + add_library(SQLite3::SQLite3 ALIAS unofficial::sqlite3::sqlite3) + else() + find_package(SQLite3 3.7.17 REQUIRED) + if(NOT TARGET SQLite3::SQLite3) # CMake < 4.3 + add_library(SQLite3::SQLite3 ALIAS SQLite::SQLite3) + endif() + endif() +endif() + +if(WITH_ZMQ) + find_package(ZeroMQ 4.0.0 MODULE REQUIRED) +endif() + +if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS) + find_package(Libmultiprocess REQUIRED COMPONENTS Lib) + find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin + NAMES Libmultiprocess + ) +endif() + +if(WITH_USDT) + find_package(USDT MODULE REQUIRED) +endif() + +if(BUILD_GUI) + set(qt_components Core Gui Widgets LinguistTools) + if(ENABLE_WALLET) + list(APPEND qt_components Network) + endif() + if(WITH_DBUS) + list(APPEND qt_components DBus) + set(USE_DBUS TRUE) + endif() + if(BUILD_GUI_TESTS) + list(APPEND qt_components Test) + endif() + find_package(Qt 6.2 MODULE REQUIRED + COMPONENTS ${qt_components} + ) + unset(qt_components) + if(WITH_QRENCODE) + find_package(QRencode MODULE REQUIRED) + set(USE_QRCODE TRUE) + endif() +endif() + include(TryAppendCXXFlags) include(TryAppendLinkerFlag) @@ -335,12 +387,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") endif() endif() -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) -target_link_libraries(core_interface INTERFACE - Threads::Threads -) - # Define sanitize_interface with -fsanitize flags intended to apply to all # libraries and executables. add_library(sanitize_interface INTERFACE) @@ -410,13 +456,6 @@ if(BUILD_FUZZ_BINARY) target_link_libraries(fuzzer_interface INTERFACE ${FUZZ_LIBS}) endif() -include(AddBoostIfNeeded) -add_boost_if_needed() - -if(BUILD_DAEMON OR BUILD_GUI OR BUILD_CLI OR BUILD_TESTS OR BUILD_BENCH OR BUILD_FUZZ_BINARY) - find_package(Libevent 2.1.8 MODULE REQUIRED) -endif() - include(cmake/introspection.cmake) include(cmake/ccache.cmake) @@ -439,6 +478,7 @@ else() try_append_cxx_flags("-Wall" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wextra" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wgnu" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wcovered-switch-default" TARGET warn_interface SKIP_LINK) # Some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. try_append_cxx_flags("-Wformat -Wformat-security" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wvla" TARGET warn_interface SKIP_LINK) @@ -461,6 +501,8 @@ else() try_append_cxx_flags("-Wself-assign" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wbidi-chars=any" TARGET warn_interface SKIP_LINK) try_append_cxx_flags("-Wundef" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wleading-whitespace=spaces" TARGET warn_interface SKIP_LINK) + try_append_cxx_flags("-Wtrailing-whitespace=any" TARGET warn_interface SKIP_LINK) # Some compilers (gcc) ignore unknown -Wno-* options, but warn about all # unknown options if any other warning is produced. Test the -Wfoo case, and @@ -479,20 +521,14 @@ configure_file(contrib/filter-lcov.py filter-lcov.py USE_SOURCE_PERMISSIONS COPY # Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. try_append_cxx_flags("-fno-extended-identifiers" TARGET core_interface SKIP_LINK) -# Avoiding the `-ffile-prefix-map` compiler option because it implies -# `-fcoverage-prefix-map` on Clang or `-fprofile-prefix-map` on GCC, -# which can cause issues with coverage builds, particularly when using -# Clang in the OSS-Fuzz environment due to its use of other options -# and a third party script, or with GCC. -try_append_cxx_flags("-fdebug-prefix-map=A=B" TARGET core_interface SKIP_LINK - IF_CHECK_PASSED "-fdebug-prefix-map=${PROJECT_SOURCE_DIR}/src=." -) +# Set `-fmacro-prefix-map`, so that source file names are expanded without the +# src prefix. try_append_cxx_flags("-fmacro-prefix-map=A=B" TARGET core_interface SKIP_LINK IF_CHECK_PASSED "-fmacro-prefix-map=${PROJECT_SOURCE_DIR}/src=." ) -# Currently all versions of gcc are subject to a class of bugs, see the -# gccbug_90348 test case (only reproduces on GCC 11 and earlier) and +# GCC versions 13.2 (and earlier) are subject to a class of bugs, see +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348 and the meta bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111843. To work around that, set # -fstack-reuse=none for all gcc builds. (Only gcc understands this flag). try_append_cxx_flags("-fstack-reuse=none" TARGET core_interface) @@ -570,19 +606,6 @@ if(REDUCE_EXPORTS) try_append_linker_flag("-Wl,-no_exported_symbols" VAR CMAKE_EXE_LINKER_FLAGS) endif() -if(WERROR) - if(MSVC) - set(werror_flag "/WX") - else() - set(werror_flag "-Werror") - endif() - try_append_cxx_flags(${werror_flag} TARGET core_interface SKIP_LINK RESULT_VAR compiler_supports_werror) - if(NOT compiler_supports_werror) - message(FATAL_ERROR "WERROR set but ${werror_flag} is not usable.") - endif() - unset(werror_flag) -endif() - # Prefer Unix-style package components over frameworks on macOS. # This improves compatibility with Python version managers. set(Python3_FIND_FRAMEWORK LAST CACHE STRING "") @@ -619,19 +642,6 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.29) set(CMAKE_SKIP_TEST_ALL_DEPENDENCY FALSE) endif() -# TODO: The `CMAKE_SKIP_BUILD_RPATH` variable setting can be deleted -# in the future after reordering Guix script commands to -# perform binary checks after the installation step. -# Relevant discussions: -# - https://github.com/hebasto/bitcoin/pull/236#issuecomment-2183120953 -# - https://github.com/bitcoin/bitcoin/pull/30312#issuecomment-2191235833 -# NetBSD always requires runtime paths to be set for executables. -if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -else() - set(CMAKE_SKIP_BUILD_RPATH TRUE) - set(CMAKE_SKIP_INSTALL_RPATH TRUE) -endif() add_subdirectory(test) add_subdirectory(doc) @@ -639,7 +649,6 @@ add_subdirectory(src) include(Maintenance) setup_split_debug_script() -add_maintenance_targets() add_windows_deploy_target() add_macos_deploy_target() @@ -668,6 +677,7 @@ message(" bitcoin-util ........................ ${BUILD_UTIL}") message(" bitcoin-wallet ...................... ${BUILD_WALLET_TOOL}") message(" bitcoin-chainstate (experimental) ... ${BUILD_UTIL_CHAINSTATE}") message(" libbitcoinkernel (experimental) ..... ${BUILD_KERNEL_LIB}") +message(" kernel-test (experimental) .......... ${BUILD_KERNEL_TEST}") message("Optional features:") message(" wallet support ...................... ${ENABLE_WALLET}") message(" external signer ..................... ${ENABLE_EXTERNAL_SIGNER}") @@ -682,6 +692,7 @@ else() set(ipc_status OFF) endif() message(" IPC ................................. ${ipc_status}") +message(" Embedded ASMap ...................... ${WITH_EMBEDDED_ASMAP}") message(" USDT tracing ........................ ${WITH_USDT}") message(" QR code (GUI) ....................... ${WITH_QRENCODE}") message(" DBus (GUI) .......................... ${WITH_DBUS}") @@ -700,7 +711,7 @@ message("Cross compiling ....................... ${cross_status}") message("C++ compiler .......................... ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}, ${CMAKE_CXX_COMPILER}") include(FlagsSummary) flags_summary() -message("Treat compiler warnings as errors ..... ${WERROR}") +message("Treat compiler warnings as errors ..... ${CMAKE_COMPILE_WARNING_AS_ERROR}") message("Use ccache for compiling .............. ${WITH_CCACHE}") message("\n") if(configure_warnings) @@ -709,6 +720,7 @@ if(configure_warnings) message(WARNING "${warning}") endforeach() message(" ******\n") + message(AUTHOR_WARNING "Warnings have been encountered!") endif() # We want all build properties to be encapsulated properly. diff --git a/CMakePresets.json b/CMakePresets.json index d478af3e70b1..a7103e2902d9 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -2,35 +2,37 @@ "version": 3, "configurePresets": [ { - "name": "vs2022", - "displayName": "Build using 'Visual Studio 17 2022' generator and 'x64-windows' triplet", + "name": "vs2026", + "displayName": "Build using 'Visual Studio 18 2026' generator and 'x64-windows' triplet", "condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Windows" }, - "generator": "Visual Studio 17 2022", + "generator": "Visual Studio 18 2026", "architecture": "x64", "toolchainFile": "$env{VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake", "cacheVariables": { "VCPKG_TARGET_TRIPLET": "x64-windows", - "BUILD_GUI": "ON" + "BUILD_GUI": "ON", + "WITH_ZMQ": "ON" } }, { - "name": "vs2022-static", - "displayName": "Build using 'Visual Studio 17 2022' generator and 'x64-windows-static' triplet", + "name": "vs2026-static", + "displayName": "Build using 'Visual Studio 18 2026' generator and 'x64-windows-static' triplet", "condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Windows" }, - "generator": "Visual Studio 17 2022", + "generator": "Visual Studio 18 2026", "architecture": "x64", "toolchainFile": "$env{VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake", "cacheVariables": { "VCPKG_TARGET_TRIPLET": "x64-windows-static", - "BUILD_GUI": "ON" + "BUILD_GUI": "ON", + "WITH_ZMQ": "ON" } }, { diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f42f0be3aa6..0505b6b1b7d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,38 +19,18 @@ Getting Started New contributors are very welcome and needed. -Reviewing and testing is highly valued and the most effective way you can contribute -as a new contributor. It also will teach you much more about the code and -process than opening pull requests. Please refer to the [peer review](#peer-review) -section below. +In-depth reviewing and testing are the bottleneck of the project, and are the +most effective way anyone can start to contribute. It will teach you much more +about the code and process than opening pull requests, and may help you uncover +related issues and follow-ups to contribute code for. Please refer to the [peer +review](#peer-review) section below. Before you start contributing, familiarize yourself with the Bitcoin Core build system and tests. Refer to the documentation in the repository on how to build Bitcoin Core and how to run the unit tests, functional tests, and fuzz tests. -There are many open issues of varying difficulty waiting to be fixed. -If you're looking for somewhere to start contributing, check out the -[good first issue](https://github.com/bitcoin/bitcoin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) -list or changes that are -[up for grabs](https://github.com/bitcoin/bitcoin/issues?utf8=%E2%9C%93&q=label%3A%22Up+for+grabs%22). -Some of them might no longer be applicable. So if you are interested, but -unsure, you might want to leave a comment on the issue first. - You may also participate in the [Bitcoin Core PR Review Club](https://bitcoincore.reviews/). -### Good First Issue Label - -The purpose of the `good first issue` label is to highlight which issues are -suitable for a new contributor without a deep understanding of the codebase. - -However, good first issues can be solved by anyone. If they remain unsolved -for a longer time, a frequent contributor might address them. - -You do not need to request permission to start working on an issue. However, -you are encouraged to leave a comment if you are planning to work on it. This -will help other contributors monitor which issues are actively being addressed -and is also an effective way to request assistance if and when you need it. - Communication Channels ---------------------- @@ -78,6 +58,13 @@ The codebase is maintained using the "contributor workflow" where everyone without exception contributes patch proposals using "pull requests" (PRs). This facilitates social contribution, easy testing and peer review. +Pull request authors must fully and confidently understand their own changes +and must have tested them. Contributors should mention which tests cover their +changes, or include the manual steps they used to confirm the change. +Contributors are expected to be prepared to clearly motivate and explain their +changes. If there is doubt, the pull request may be closed. +Please refer to the [peer review](#peer-review) section below for more details. + To contribute a patch, the workflow is as follows: 1. Fork repository ([only for the first time](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo)) @@ -139,7 +126,7 @@ about Git. ### Creating the Pull Request The title of the pull request should be prefixed by the component or area that -the pull request affects. Valid areas as: +the pull request affects. Valid areas are: - `consensus` for changes to consensus critical code - `doc` for changes to the documentation @@ -177,7 +164,7 @@ subsequent comment to the PR. ### Translation changes Note that translations should not be submitted as pull requests. Please see -[Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) +[Translation Process](/doc/translation_process.md) for more information on helping with translations. ### Work in Progress Changes and Requests for Comments @@ -338,6 +325,11 @@ reviewers that the changes warrant the review effort, and if reviewers are "Concept NACK'ing" the PR, the author may need to present arguments and/or do research backing their suggested changes. +Moreover, if there is reasonable doubt that the pull request author does not +fully understand the changes they are submitting themselves, or if it becomes +clear that they have not tested the changes on a basic level themselves, the +pull request may be closed immediately. + #### Conceptual Review A review can be a conceptual review, where the reviewer leaves a comment diff --git a/COPYING b/COPYING index 23dc5e905b60..89960cbf2f22 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2025 The Bitcoin Core developers -Copyright (c) 2009-2025 Bitcoin Developers +Copyright (c) 2009-2026 The Bitcoin Core developers +Copyright (c) 2009-2026 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 000000000000..539a37d2ff4b --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,2 @@ +set(CTEST_SUBMIT_URL "https://my.cdash.org/submit.php?project=bitcoin-core") +set(CTEST_NIGHTLY_START_TIME "00:00:00 UTC") diff --git a/SECURITY.md b/SECURITY.md index fd4c61d176dc..5a63a762ac2d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,5 +16,6 @@ The following keys may be used to communicate sensitive information to developer | Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | | Michael Ford | E777 299F C265 DD04 7930 70EB 944D 35F9 AC3D B76A | | Ava Chow | 1528 1230 0785 C964 44D3 334D 1756 5732 E08E 5E41 | +| Niklas Gögge | 2CBB F208 E594 BF43 9B5F 276C 7465 CFFF 6793 242E | You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. diff --git a/ci/README.md b/ci/README.md index 81e048ce687c..33c623dc2afa 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,11 +20,30 @@ requires `bash`, `docker`, and `python3` to be installed. To run on different ar sudo apt install bash docker.io python3 qemu-user-static ``` -It is recommended to run the ci system in a clean env. To run the test stage -with a specific configuration, +For some sanitizer builds, the kernel's address-space layout randomization +(ASLR) entropy can cause sanitizer shadow memory mappings to fail. When running +the CI locally you may need to reduce that entropy by running: ``` -env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' +sudo sysctl -w vm.mmap_rnd_bits=28 +``` + +To run a test that requires emulating a CPU architecture different from the +host, we may rely on the container environment recognizing foreign executables +and automatically running them using `qemu`. The following sets us up to do so +(also works for `podman`): + +``` +docker run --rm --privileged docker.io/multiarch/qemu-user-static --reset -p yes +``` + +It is recommended to run the CI system in a clean environment. The `env -i` +command below ensures that *only* specified environment variables are propagated +into the local CI. +To run the test stage with a specific configuration: + +``` +env -i HOME="$HOME" PATH="$PATH" USER="$USER" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` ## Configurations @@ -43,7 +62,7 @@ It is also possible to force a specific configuration without modifying the file. For example, ``` -env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh' +env -i HOME="$HOME" PATH="$PATH" USER="$USER" MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` The files starting with `0n` (`n` greater than 0) are the scripts that are run @@ -61,20 +80,22 @@ trigger cache-invalidation and rebuilds as necessary. To configure the primary repository, follow these steps: -1. Register with [Cirrus Runners](https://cirrus-runners.app/) and purchase runners. -2. Install the Cirrus Runners GitHub app against the GitHub organization. +1. Register with [WarpBuild](https://www.warpbuild.com/) and purchase runners. +2. Install the WarpBuild GitHub app against the GitHub organization. 3. Enable organisation-level runners to be used in public repositories: 1. `Org settings -> Actions -> Runner Groups -> Default -> Allow public repos` 4. Permit the following actions to run: - 1. cirruslabs/cache/restore@\* - 1. cirruslabs/cache/save@\* - 1. docker/setup-buildx-action@\* + 1. actions/cache/restore@\* + 1. actions/cache/save@\* 1. actions/github-script@\* + 1. docker/setup-buildx-action@\* + 1. warpbuilds/cache/restore@\* + 1. warpbuilds/cache/save@\* ### Forked repositories When used in a fork the CI will run on GitHub's free hosted runners by default. -In this case, due to GitHub's 10GB-per-repo cache size limitations caches will be frequently evicted and missed, but the workflows will run (slowly). +In this case, GitHub's cache size limitations may cause caches to be frequently evicted and missed, but the workflows will run (slowly). -It is also possible to use your own Cirrus Runners in your own fork with an appropriate patch to the `REPO_USE_CIRRUS_RUNNERS` variable in ../.github/workflows/ci.yml -NB that Cirrus Runners only work at an organisation level, therefore in order to use your own Cirrus Runners, *the fork must be within your own organisation*. +It is also possible to use your own WarpBuild Runners in your own fork by replacing the references to `bitcoin/bitcoin` in `../.github/workflows/ci.yml` with the name of your fork (e.g. `your-org/bitcoin`). +NB that WarpBuild Runners only work at an organisation level, therefore in order to use your own WarpBuild Runners, *the fork must be within your own organisation*. diff --git a/ci/lint.py b/ci/lint.py new file mode 100755 index 000000000000..6c7e0e8ad419 --- /dev/null +++ b/ci/lint.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + + +def run(cmd, **kwargs): + print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) + try: + return subprocess.run(cmd, **kwargs) + except Exception as e: + sys.exit(str(e)) + + +def get_worktree_mounts(repo_root): + git_path = repo_root / ".git" + if not git_path.is_file(): + return [] + content = git_path.read_text().strip() + if not content.startswith("gitdir: "): + return [] + gitdir = (repo_root / content.removeprefix("gitdir: ")).resolve() + main_gitdir = gitdir.parent.parent + return [ + f"--volume={gitdir}:{gitdir}", + f"--volume={main_gitdir}:{main_gitdir}", + ] + + +def main(): + repo_root = Path(__file__).resolve().parent.parent + is_ci = os.environ.get("GITHUB_ACTIONS") == "true" + container = "bitcoin-linter" + + build_cmd = [ + "docker", + "buildx", + "build", + "--platform=linux", + f"--tag={container}", + *shlex.split(os.environ.get("DOCKER_BUILD_CACHE_ARG", "")), + f"--file={repo_root}/ci/lint_imagefile", + str(repo_root), + ] + if run(build_cmd, check=False).returncode != 0: + if is_ci: + print("Retry building image after failure") + time.sleep(3) + run(build_cmd) + + extra_env = [] + if is_ci: + if os.environ.get("GITHUB_EVENT_NAME") == "pull_request": + extra_env = ["--env", "LINT_CI_IS_PR=1"] + elif os.environ.get("GITHUB_REPOSITORY") == "bitcoin/bitcoin": + extra_env = ["--env", "LINT_CI_SANITY_CHECK_COMMIT_SIG=1"] + + run( + [ + "docker", + "run", + "--rm", + *extra_env, + f"--volume={repo_root}:/bitcoin", + *get_worktree_mounts(repo_root), + *([] if is_ci else ["-it"]), + container, + "./ci/lint/06_script.sh", + *sys.argv[1:], + ] + ) + + +if __name__ == "__main__": + main() diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 75f891105c05..9c2020a9a15d 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -8,7 +8,8 @@ export LC_ALL=C set -o errexit -o pipefail -o xtrace -export CI_RETRY_EXE="/ci_retry --" +export DEBIAN_FRONTEND=noninteractive +export CI_RETRY_EXE="/ci_retry" pushd "/" @@ -18,42 +19,25 @@ ${CI_RETRY_EXE} apt-get update # - curl/xz-utils (to install shellcheck) # - git (used in many lint scripts) # - gpg (used by verify-commits) -${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg - -PYTHON_PATH="/python_build" -if [ ! -d "${PYTHON_PATH}/bin" ]; then - ( - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/pyenv/pyenv.git - cd pyenv/plugins/python-build || exit 1 - ./install.sh - ) - # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment - ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ - clang - env CC=clang python-build "$(cat "/.python-version")" "${PYTHON_PATH}" -fi -export PATH="${PYTHON_PATH}/bin:${PATH}" +# - moreutils (used by scripted-diff) +${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg moreutils + +# Install Python and create venv using uv (reads version from .python-version) +uv venv /python_env + +export PATH="/python_env/bin:${PATH}" command -v python3 python3 --version -${CI_RETRY_EXE} pip3 install \ - codespell==2.4.1 \ - lief==0.16.6 \ - mypy==1.4.1 \ - pyzmq==25.1.0 \ - ruff==0.5.5 \ - vulture==2.6 +uv pip install --python /python_env --requirements /ci/lint/requirements.txt SHELLCHECK_VERSION=v0.11.0 -curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ +curl --fail -L "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.$(uname --machine).tar.xz" | \ tar --xz -xf - --directory /tmp/ mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ -MLC_VERSION=v1 -MLC_BIN=mlc-x86_64-linux -curl -sL "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/${MLC_BIN}" -o "/usr/bin/mlc" +MLC_VERSION=v1.2.0 +curl --fail -L "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/mlc-$(uname --machine)-linux" -o "/usr/bin/mlc" chmod +x /usr/bin/mlc popd || exit diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 6d637c2a438b..a0f2dc88bc47 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -6,19 +6,25 @@ export LC_ALL=C -set -ex +set -o errexit -o pipefail -o xtrace -if [ -n "$CIRRUS_PR" ]; then +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the mounted bitcoin src dir. +git config --global --add safe.directory /bitcoin + +export PATH="/python_env/bin:${PATH}" + +if [ -n "${LINT_CI_IS_PR}" ]; then export COMMIT_RANGE="HEAD~..HEAD" if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then - echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch." + echo "Error: The top commit must be a merge commit, usually the remote 'pull//merge' branch." false fi fi -RUST_BACKTRACE=1 cargo run --manifest-path "./test/lint/test_runner/Cargo.toml" +RUST_BACKTRACE=1 cargo run --manifest-path "./test/lint/test_runner/Cargo.toml" -- "$@" -if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then +if [ "${LINT_CI_SANITY_CHECK_COMMIT_SIG}" = "1" ] ; then # Sanity check only the last few commits to get notified of missing sigs, # missing keys, or expired keys. Usually there is only one new merge commit # per push on the master branch and a few commits on release branches, so diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh deleted file mode 100755 index 84e60be2917a..000000000000 --- a/ci/lint/container-entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit/. - -export LC_ALL=C - -# Fixes permission issues when there is a container UID/GID mismatch with the owner -# of the mounted bitcoin src dir. -git config --global --add safe.directory /bitcoin - -export PATH="/python_build/bin:${PATH}" - -if [ -z "$1" ]; then - bash -ic "./ci/lint/06_script.sh" -else - exec "$@" -fi diff --git a/ci/lint/requirements.txt b/ci/lint/requirements.txt new file mode 100644 index 000000000000..e8abf041dc76 --- /dev/null +++ b/ci/lint/requirements.txt @@ -0,0 +1,3 @@ +lief==0.17.5 +mypy==1.19.1 +pyzmq==27.1.0 diff --git a/ci/lint_imagefile b/ci/lint_imagefile index b32380769de5..a6c49a7c8619 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -6,19 +6,18 @@ FROM mirror.gcr.io/ubuntu:24.04 -ENV DEBIAN_FRONTEND=noninteractive -ENV LC_ALL=C.UTF-8 +# Pin uv and ruff to minor version to avoid breaking changes +# https://docs.astral.sh/uv/reference/policies/versioning/ +# https://docs.astral.sh/ruff/versioning/ +COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/ruff:0.15 /ruff /bin/ COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version -COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh +COPY ./ci/lint/requirements.txt /ci/lint/requirements.txt COPY ./ci/lint/01_install.sh /install.sh RUN /install.sh && \ - echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ - chmod 755 /entrypoint.sh && \ rm -rf /var/lib/apt/lists/* - WORKDIR /bitcoin -ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/retry/retry b/ci/retry/retry index 3c06519dbdfe..7942b6c4f09b 100755 --- a/ci/retry/retry +++ b/ci/retry/retry @@ -1,14 +1,9 @@ #!/usr/bin/env bash -GETOPT_BIN=$IN_GETOPT_BIN -GETOPT_BIN=${GETOPT_BIN:-getopt} - __sleep_amount() { if [ -n "$constant_sleep" ]; then sleep_time=$constant_sleep else - #TODO: check for awk - #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"` fi } @@ -46,7 +41,6 @@ retry() P="$1" for param in "${@:2}"; do P="$P '$param'"; done - #TODO: replace single quotes in each arg with '"'"' ? export RETRY_ATTEMPT=$attempts bash -c "$P" return_code=$? @@ -71,93 +65,10 @@ retry() exit $return_code } -# If we're being sourced, don't worry about such things -if [ "$BASH_SOURCE" == "$0" ]; then - # Prints the help text - help() - { - local retry=$(basename $0) - cat < /dev/null - if [[ $? -ne 4 ]]; then - echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt." - exit 1 - fi - - OPTIONS=vt:s:m:x:f: - LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail: - - PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@") - if [[ $? -ne 0 ]]; then - # e.g. $? == 1 - # then getopt has complained about wrong arguments to stdout - exit 2 - fi - # read getopt’s output this way to handle the quoting right: - eval set -- "$PARSED" - max_tries=10 min_sleep=0.3 max_sleep=60.0 constant_sleep= fail_script= - # now enjoy the options in order and nicely split until we see -- - while true; do - case "$1" in - -v|--verbose) - VERBOSE=true - shift - ;; - -t|--tries) - max_tries="$2" - shift 2 - ;; - -s|--sleep) - constant_sleep="$2" - shift 2 - ;; - -m|--min) - min_sleep="$2" - shift 2 - ;; - -x|--max) - max_sleep="$2" - shift 2 - ;; - -f|--fail) - fail_script="$2" - shift 2 - ;; - --) - shift - break - ;; - *) - echo "Programming error" - exit 3 - ;; - esac - done - retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@" - -fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index a4293fb80571..4ca660a5d69e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -set -ex +set -o errexit -o nounset -o pipefail -o xtrace # The source root dir, usually from git, usually read-only. # The ci system copies this folder. @@ -20,24 +20,18 @@ export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}" # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} # A folder for the ci system to put temporary files (build result, datadirs for tests, ...) +# The name contains a space and a non-ASCII symbol to confirm the build and +# tests handle word-splitting and UTF8 correctly. # This folder only exists on the ci guest. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} -# A folder for the ci system to put executables. -# This folder only exists on the ci guest. -export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch_ ₿🧪_} echo "Setting specific values in env" -if [ -n "${FILE_ENV}" ]; then - set -o errexit; - # shellcheck disable=SC1090 - source "${FILE_ENV}" -fi +# shellcheck disable=SC1090 +source "${FILE_ENV}" echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j$(if command -v nproc > /dev/null 2>&1; then nproc; else sysctl -n hw.logicalcpu; fi)} -# Whether to prefer BusyBox over GNU utilities -export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} @@ -53,7 +47,7 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} # See man 7 debconf export DEBIAN_FRONTEND=noninteractive -export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-500M} +export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-2G} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. @@ -64,10 +58,10 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake ninja-build} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3-dev rsync git procps bison e2fsprogs cmake ninja-build} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} -export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} +export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry"} # The --platform argument used with `docker build` and `docker run`. export CI_IMAGE_PLATFORM=${CI_IMAGE_PLATFORM:-"linux"} # Force linux, but use native arch by default diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index ce019784f908..71b519dab189 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -8,15 +8,17 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf export DPKG_ADD_ARCH="armhf" -export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PACKAGES="python3-pip python3-zmq g++-arm-linux-gnueabihf libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PIP_PACKAGES="--break-system-packages pycapnp" export CONTAINER_NAME=ci_arm_linux -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Check that https://packages.ubuntu.com/noble/g++-arm-linux-gnueabihf (version 13.x, similar to guix) can cross-compile +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/g++-arm-linux-gnueabihf (version 14.x, similar to guix) can cross-compile export CI_IMAGE_PLATFORM="linux/arm64" -export USE_BUSY_BOX=true -export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false export GOAL="install" export CI_LIMIT_STACK_SIZE=1 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" +export BITCOIN_CONFIG=" \ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized' \ +" diff --git a/ci/test/00_setup_env_freebsd_cross.sh b/ci/test/00_setup_env_freebsd_cross.sh new file mode 100755 index 000000000000..1be28a22c643 --- /dev/null +++ b/ci/test/00_setup_env_freebsd_cross.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_freebsd_cross +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export APT_LLVM_V="22" +export FREEBSD_VERSION=15.0 +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} lld" +export HOST=x86_64-unknown-freebsd +export DEP_OPTS="build_CC=clang build_CXX=clang++ AR=llvm-ar-${APT_LLVM_V} STRIP=llvm-strip-${APT_LLVM_V} NM=llvm-nm-${APT_LLVM_V} RANLIB=llvm-ranlib-${APT_LLVM_V}" +export GOAL="install" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ + -DWITH_USDT=OFF \ +" +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_no_ipc.sh b/ci/test/00_setup_env_i686_no_ipc.sh index 4da38d9bb999..d8df8100fe56 100755 --- a/ci/test/00_setup_env_i686_no_ipc.sh +++ b/ci/test/00_setup_env_i686_no_ipc.sh @@ -8,16 +8,16 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_no_multiprocess -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CI_IMAGE_PLATFORM="linux/amd64" -export PACKAGES="llvm clang g++-multilib" +export CI_CONTAINER_CAP="--security-opt seccomp=unconfined" +export PACKAGES="g++-multilib" export DEP_OPTS="DEBUG=1 NO_IPC=1" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 -export TEST_RUNNER_EXTRA="--v2transport --usecli" export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DENABLE_IPC=OFF \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER='clang;-m32' \ - -DCMAKE_CXX_COMPILER='clang++;-m32' \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \ " diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index a0d9082f4b9f..63d89fb9b927 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -9,12 +9,16 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" -export HOST=x86_64-apple-darwin +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=arm64-apple-darwin export PACKAGES="clang lld llvm zip" -export XCODE_VERSION=15.0 -export XCODE_BUILD_ID=15A240d +export XCODE_VERSION=26.1.1 +export XCODE_BUILD_ID=17B100 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DWITH_USDT=OFF \ + -DREDUCE_EXPORTS=ON \ +" diff --git a/ci/test/00_setup_env_mac_cross_intel.sh b/ci/test/00_setup_env_mac_cross_intel.sh new file mode 100755 index 000000000000..4b07e14b8195 --- /dev/null +++ b/ci/test/00_setup_env_mac_cross_intel.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} + +export CONTAINER_NAME=ci_macos_cross_intel +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/clang (version 19, similar to guix) can cross-compile +export HOST=x86_64-apple-darwin +export PACKAGES="clang lld llvm zip" +export XCODE_VERSION=26.1.1 +export XCODE_BUILD_ID=17B100 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DWITH_USDT=OFF \ + -DREDUCE_EXPORTS=ON \ +" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 41a3bc458777..d59c5ae65c55 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -6,14 +6,17 @@ export LC_ALL=C.UTF-8 -# Homebrew's python@3.12 is marked as externally managed (PEP 668). -# Therefore, `--break-system-packages` is needed. export CONTAINER_NAME="ci_mac_native" # macos does not use a container, but the env var is needed for logging -export PIP_PACKAGES="--break-system-packages zmq" +export PIP_PACKAGES="--break-system-packages pycapnp pyzmq" export GOAL="install deploy" export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DWITH_USDT=OFF \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000' \ +" export BITCOIN_CMD="bitcoin -m" # Used in functional tests diff --git a/ci/test/00_setup_env_mac_native_fuzz.sh b/ci/test/00_setup_env_mac_native_fuzz.sh index a8010c7aecda..ea074e47732f 100755 --- a/ci/test/00_setup_env_mac_native_fuzz.sh +++ b/ci/test/00_setup_env_mac_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME="ci_mac_native_fuzz" # macos does not use a container, but the env var is needed for logging export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'" +export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000' -DAPPEND_CPPFLAGS='-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG'" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_native_alpine_musl.sh b/ci/test/00_setup_env_native_alpine_musl.sh new file mode 100755 index 000000000000..3801dfec403c --- /dev/null +++ b/ci/test/00_setup_env_native_alpine_musl.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_alpine_musl +export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.24" +export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3-dev py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers" +export PIP_PACKAGES="--break-system-packages pyzmq pycapnp" +export DEP_OPTS="DEBUG=1" +export GOAL="install" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_BUILD_TYPE=Debug \ +" +export TEST_RUNNER_EXTRA="--v2transport --usecli --extended --exclude feature_dbcrash" # Run extended tests under --usecli and --v2transport, but exclude the very slow dbcrash +export BITCOIN_CMD="bitcoin -m" # Used in functional tests diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index e785f6ab51fb..7e28b05e0b02 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Only install BCC tracing packages in CI. Container has to match the host for BCC to work. -if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then +if [[ "${INSTALL_BCC_TRACING_TOOLS:-}" == "true" ]]; then # Required for USDT functional tests to run BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)" export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw" @@ -19,19 +19,20 @@ else fi export CONTAINER_NAME=ci_native_asan -export APT_LLVM_V="21" -export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" +export APT_LLVM_V="22" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev mold python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto python3-pip" export PIP_PACKAGES="--break-system-packages pycapnp" export NO_DEPENDS=1 export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export BITCOIN_CONFIG="\ - -DWITH_USDT=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON \ + --preset=dev-mode \ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \ + -DCMAKE_EXE_LINKER_FLAGS='-fuse-ld=mold' \ -DAPPEND_CXXFLAGS='-std=c++23' \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ " diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_centos.sh deleted file mode 100755 index 998ddaf45f82..000000000000 --- a/ci/test/00_setup_env_native_centos.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" -export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make ninja-build git python3 python3-pip which patch xz procps-ng rsync coreutils bison e2fsprogs cmake dash" -export PIP_PACKAGES="pyzmq pycapnp" -export DEP_OPTS="DEBUG=1" -export GOAL="install" -export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON \ - -DBUILD_GUI=ON \ - -DREDUCE_EXPORTS=ON \ - -DCMAKE_BUILD_TYPE=Debug \ -" -export BITCOIN_CMD="bitcoin -m" # Used in functional tests diff --git a/ci/test/00_setup_env_native_chimera_lto.sh b/ci/test/00_setup_env_native_chimera_lto.sh new file mode 100755 index 000000000000..1cb983bf5c4e --- /dev/null +++ b/ci/test/00_setup_env_native_chimera_lto.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +# This config is experimental, and may not be reproducible, given +# the use of a rolling distro. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_chimera_musl +export CI_IMAGE_NAME_TAG="mirror.gcr.io/chimeralinux/chimera" +export CI_BASE_PACKAGES="ccache chimerautils chimerautils-extra clang cmake curl e2fsprogs git gmake gtar linux-headers procps python-devel python-pip rsync util-linux util-linux-lscpu" +export PIP_PACKAGES="--break-system-packages pyzmq pycapnp" +# NO_QT=1 because Qt needs various patches: https://github.com/chimera-linux/cports/tree/master/main/qt6-qtbase/patches +export DEP_OPTS="build_CC=clang build_CXX=clang++ build_TAR=gtar AR=llvm-ar CC=clang CXX=clang++ NM=llvm-nm RANLIB=llvm-ranlib STRIP=llvm-strip NO_QT=1" +export GOAL="install" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DBUILD_GUI=OFF \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ +" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index b3ad36a8bab7..bb203a728509 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,10 +6,10 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_fuzz -export APT_LLVM_V="21" -export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libevent-dev libboost-dev libsqlite3-dev libcapnp-dev capnproto" +export APT_LLVM_V="22" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libboost-dev libsqlite3-dev libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index c71772b8e2cb..c35a93622df8 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,13 +6,15 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev" export DEP_OPTS="DEBUG=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="all" # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. @@ -22,7 +24,7 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_FLAGS_DEBUG='' \ -DCMAKE_CXX_FLAGS_DEBUG='' \ -DBUILD_FOR_FUZZING=ON \ - -DSANITIZERS=fuzzer,memory \ + -DSANITIZERS=memory \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE' \ " export USE_INSTRUMENTED_LIBCPP="MemoryWithOrigins" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index f1d99f818194..b6442c78fb86 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -6,9 +6,9 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="libevent-dev libboost-dev libsqlite3-dev valgrind libcapnp-dev capnproto" +export PACKAGES="clang llvm libclang-rt-dev libboost-dev libsqlite3-dev valgrind libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -17,5 +17,6 @@ export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="all" export BITCOIN_CONFIG="\ -DBUILD_FOR_FUZZING=ON \ - -DCMAKE_CXX_FLAGS='-Wno-error=array-bounds' \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ " diff --git a/ci/test/00_setup_env_native_iwyu.sh b/ci/test/00_setup_env_native_iwyu.sh new file mode 100755 index 000000000000..14c367bb31cd --- /dev/null +++ b/ci/test/00_setup_env_native_iwyu.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2023-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" # To build codegen, CMake must be 3.31 or newer. +export CONTAINER_NAME=ci_native_iwyu +export IWYU_LLVM_V="22" +export APT_LLVM_V="${IWYU_LLVM_V}" +export PACKAGES="clang-${IWYU_LLVM_V} clang-format-${IWYU_LLVM_V} libclang-${IWYU_LLVM_V}-dev llvm-${IWYU_LLVM_V}-dev jq libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=false +export RUN_CHECK_DEPS=false +export RUN_IWYU=true +export GOAL="codegen" +export BITCOIN_CONFIG="\ + --preset dev-mode -DBUILD_GUI=OFF \ + -DCMAKE_C_COMPILER=clang-${IWYU_LLVM_V} \ + -DCMAKE_CXX_COMPILER=clang++-${IWYU_LLVM_V} \ +" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 4ff82614e9de..1d005abbd605 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -6,8 +6,8 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" -export APT_LLVM_V="21" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" @@ -15,13 +15,15 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-pip" -export PIP_PACKAGES="--break-system-packages pycapnp" +export PIP_PACKAGES="--break-system-packages pycapnp pyzmq" export DEP_OPTS="DEBUG=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. # _FORTIFY_SOURCE is not compatible with MSAN. export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DBUILD_GUI=OFF \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS_DEBUG='' \ -DCMAKE_CXX_FLAGS_DEBUG='' \ diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh new file mode 100755 index 000000000000..0b21ab226a91 --- /dev/null +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_nowallet +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +# Use minimum supported python3.10 (or best-effort 3.12) and clang-17, see doc/dependencies.md +export PACKAGES="python3-zmq python3-pip clang-17 llvm-17 libc++abi-17-dev libc++-17-dev" +export PIP_PACKAGES="--break-system-packages pycapnp" +export DEP_OPTS="NO_WALLET=1 CC=clang-17 CXX='clang++-17 -stdlib=libc++'" +export GOAL="install" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ + -DENABLE_WALLET=OFF \ + -DWITH_EMBEDDED_ASMAP=OFF \ +" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh deleted file mode 100755 index 58561e552040..000000000000 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:bookworm" -# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md -export PACKAGES="python3-zmq python3-pip clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" -export PIP_PACKAGES="--break-system-packages pycapnp" -export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" -export GOAL="install" -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_SHARED_LIBS=ON" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index d9f8d3bfc908..d6af52c4bf31 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -8,19 +8,21 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:22.04" -# Use minimum supported python3.10 and gcc-11, see doc/dependencies.md -export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="CC=gcc-11 CXX=g++-11" +# Use minimum supported python3.10 and gcc-12, see doc/dependencies.md +export PACKAGES="gcc-12 g++-12 python3-zmq" +export DEP_OPTS="CC=gcc-12 CXX=g++-12" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export GOAL="install" export CI_LIMIT_STACK_SIZE=1 export DOWNLOAD_PREVIOUS_RELEASES="true" +# Use -Werror as the CMake version does not support CMAKE_COMPILE_WARNING_AS_ERROR export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON \ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_FLAGS='-funsigned-char' \ + -DCMAKE_C_FLAGS='-funsigned-char -Werror' \ -DCMAKE_C_FLAGS_DEBUG='-g2 -O2' \ - -DCMAKE_CXX_FLAGS='-funsigned-char' \ + -DCMAKE_CXX_FLAGS='-funsigned-char -Werror' \ -DCMAKE_CXX_FLAGS_DEBUG='-g2 -O2' \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \ " diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index c14e59b7289f..845a43705b78 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -6,11 +6,11 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_tidy -export TIDY_LLVM_V="20" +export TIDY_LLVM_V="22" export APT_LLVM_V="${TIDY_LLVM_V}" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -19,7 +19,7 @@ export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_USDT=ON \ + --preset dev-mode \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ -DCMAKE_C_FLAGS_RELWITHDEBINFO='-O0 -g0' \ diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index bf4b68412b2c..4f900699f10c 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" -export APT_LLVM_V="21" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" LIBCXX_FLAGS="-fsanitize=thread -nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-zmq python3-pip" @@ -16,6 +16,11 @@ export PIP_PACKAGES="--break-system-packages pycapnp" export DEP_OPTS="CC=clang CXX=clang++ CXXFLAGS='${LIBCXX_FLAGS}' NO_QT=1" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 -export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \ --DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'" +# Disable fortification with -U_FORTIFY_SOURCE to work around https://github.com/bitcoin/bitcoin/issues/30586 +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DBUILD_GUI=OFF \ + -DSANITIZERS=thread \ + -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES -U_FORTIFY_SOURCE' \ +" export USE_INSTRUMENTED_LIBCPP="Thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d6c45755df25..52b5a37bfa30 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,16 +6,20 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind python3-zmq libevent-dev libboost-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto python3-pip" +export PACKAGES="clang llvm libclang-rt-dev valgrind python3-zmq libboost-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto python3-pip" export PIP_PACKAGES="--break-system-packages pycapnp" export USE_VALGRIND=1 export NO_DEPENDS=1 # bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 -export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" +export TEST_RUNNER_EXTRA="--exclude rpc_bind --exclude feature_bind_extra" export GOAL="install" -# TODO enable GUI +# GUI disabled, because it only passes with a DEBUG=1 depends build export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=OFF \ + --preset=dev-mode \ + -DBUILD_GUI=OFF \ + -DWITH_USDT=OFF \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ " diff --git a/ci/test/00_setup_env_openbsd_cross.sh b/ci/test/00_setup_env_openbsd_cross.sh new file mode 100755 index 000000000000..3d80e28626e0 --- /dev/null +++ b/ci/test/00_setup_env_openbsd_cross.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_openbsd_cross +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export APT_LLVM_V="22" +export HOST=x86_64-unknown-openbsd +export OPENBSD_VERSION=7.9 +export OPENBSD_SDK_BASENAME="openbsd-${HOST}-${OPENBSD_VERSION}" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} lld" +export SYSROOT="--sysroot=${DEPENDS_DIR}/SDKs/${OPENBSD_SDK_BASENAME}" +export DEP_OPTS="build_CC=clang build_CXX=clang++ \ + CC='clang --target=${HOST} ${SYSROOT}' \ + CXX='clang++ --target=${HOST} ${SYSROOT} -stdlib=libc++' \ + LDFLAGS='-fuse-ld=lld' \ + AR=llvm-ar-${APT_LLVM_V} \ + NM=llvm-nm-${APT_LLVM_V} \ + OBJCOPY=llvm-objcopy-${APT_LLVM_V} \ + OBJDUMP=llvm-objdump-${APT_LLVM_V} \ + RANLIB=llvm-ranlib-${APT_LLVM_V} \ + STRIP=llvm-strip-${APT_LLVM_V}" +export GOAL="install" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DBUILD_GUI=OFF \ + -DREDUCE_EXPORTS=ON \ + -DWITH_USDT=OFF \ +" +export RUN_UNIT_TESTS="false" +export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_riscv_bare_cross.sh b/ci/test/00_setup_env_riscv_bare_cross.sh new file mode 100755 index 000000000000..b68434087dcf --- /dev/null +++ b/ci/test/00_setup_env_riscv_bare_cross.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright (c) Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_riscv_bare + +export GOAL="bitcoin_consensus bitcoin_crypto secp256k1" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" +export HOST="riscv32-unknown-elf-gcc" +export PACKAGES="autoconf automake autotools-dev curl python3 python3-pip libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex texinfo gperf libtool patchutils bc zlib1g-dev libexpat-dev ninja-build git cmake libglib2.0-dev libslirp-dev" +export BITCOIN_CONFIG="-DCMAKE_C_COMPILER=/opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc \ + -DCMAKE_CXX_COMPILER=/opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ \ + -DBUILD_KERNEL_LIB=OFF \ + -DBUILD_UTIL_CHAINSTATE=OFF \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCH=OFF \ + -DBUILD_FUZZ_BINARY=OFF \ + -DBUILD_DAEMON=OFF \ + -DBUILD_TX=OFF \ + -DBUILD_UTIL=OFF \ + -DBUILD_CLI=OFF \ + -DBUILD_BITCOIN_BIN=OFF \ + -DENABLE_WALLET=OFF \ + -DENABLE_EXTERNAL_SIGNER=OFF \ + -DENABLE_IPC=OFF \ + -DCMAKE_SYSTEM_NAME=Generic \ + -DIFADDR_LINKS_WITHOUT_LIBSOCKET=ON \ + " + +export BARE_METAL_RISCV="true" +export RUN_UNIT_TESTS="false" +export RUN_FUNCTIONAL_TESTS="false" +export NO_DEPENDS="true" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index b84d99076ea9..a7c59d769635 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -9,9 +9,13 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu export PACKAGES="python3-zmq" export CONTAINER_NAME=ci_s390x -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:26.04" export CI_IMAGE_PLATFORM="linux/s390x" -export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +# bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind --exclude feature_bind_extra" export RUN_FUNCTIONAL_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DREDUCE_EXPORTS=ON \ +" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 13e794bd6d1d..dbbf55c14781 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,18 +1,22 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-present The Bitcoin Core developers +# Copyright (c) 2025-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" # Check that https://packages.ubuntu.com/noble/g++-mingw-w64-x86-64-posix (version 13.x, similar to guix) can cross-compile -export CI_IMAGE_PLATFORM="linux/amd64" -export HOST=x86_64-w64-mingw32 -export PACKAGES="g++-mingw-w64-x86-64-posix nsis" +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/g++-mingw-w64-ucrt64 can cross-compile +export HOST=x86_64-w64-mingw32ucrt +export PACKAGES="g++-mingw-w64-ucrt64 nsis" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_GUI_TESTS=OFF \ --DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized'" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DENABLE_IPC=OFF \ + -DWITH_USDT=OFF \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized' \ +" diff --git a/ci/test/00_setup_env_win64_msvcrt.sh b/ci/test/00_setup_env_win64_msvcrt.sh new file mode 100755 index 000000000000..6c948aeb351e --- /dev/null +++ b/ci/test/00_setup_env_win64_msvcrt.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_win64_msvcrt +export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/g++-mingw-w64-x86-64-posix (version 14.x, similar to guix) can cross-compile +export HOST=x86_64-w64-mingw32 +export PACKAGES="g++-mingw-w64-x86-64-posix nsis" +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export GOAL="deploy" +export BITCOIN_CONFIG="\ + --preset=dev-mode \ + -DENABLE_IPC=OFF \ + -DWITH_USDT=OFF \ + -DREDUCE_EXPORTS=ON \ + -DCMAKE_CXX_FLAGS='-Wno-error=maybe-uninitialized' \ +" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 6944ab42b06c..466d2e3b13df 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -8,6 +8,11 @@ export LC_ALL=C.UTF-8 set -o errexit -o pipefail -o xtrace +if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then + echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1" + exit 1 +fi + CFG_DONE="${BASE_ROOT_DIR}/ci.base-install-done" # Use a global setting to remember whether this script ran to avoid running it twice if [ "$( cat "${CFG_DONE}" || true )" == "done" ]; then @@ -32,16 +37,17 @@ if [ -n "${APT_LLVM_V}" ]; then ) fi -if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - bash -c "dnf -y install epel-release" - # The ninja-build package is available in the CRB repository. - bash -c "dnf -y --allowerasing --enablerepo crb install $CI_BASE_PACKAGES $PACKAGES" +if command -v apk >/dev/null 2>&1; then + ${CI_RETRY_EXE} apk update + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apk add --no-cache $CI_BASE_PACKAGES $PACKAGES elif [ "$CI_OS_NAME" != "macos" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi ${CI_RETRY_EXE} apt-get update - ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES" + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES fi if [ -n "${APT_LLVM_V}" ]; then @@ -56,27 +62,13 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.1" /llvm-project - - if [ -n "${APT_LLVM_V}" ]; then - - cmake -G Ninja -B /clang_build/ \ - -DLLVM_ENABLE_PROJECTS="clang" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_TARGETS_TO_BUILD=Native \ - -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ - -S /llvm-project/llvm - - ninja -C /clang_build/ "$MAKEJOBS" - ninja -C /clang_build/ install-runtimes - - update-alternatives --install /usr/bin/clang++ clang++ /clang_build/bin/clang++ 100 - update-alternatives --install /usr/bin/clang clang /clang_build/bin/clang 100 - update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /clang_build/bin/llvm-symbolizer 100 - fi + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-22.1.7" /llvm-project +# LLVM is configured with LIBCXXABI_USE_LLVM_UNWINDER=OFF, +# because libunwind doesn't handle exceptions under MSAN. +# https://github.com/llvm/llvm-project/issues/84348 cmake -G Ninja -B /cxx_build/ \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_USE_SANITIZER="${USE_INSTRUMENTED_LIBCPP}" \ -DCMAKE_C_COMPILER=clang \ @@ -95,9 +87,21 @@ if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then rm -rf /llvm-project fi -if [[ "${RUN_TIDY}" == "true" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use - cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use +if [[ ${BARE_METAL_RISCV} == "true" ]]; then + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/riscv-collab/riscv-gnu-toolchain -b 2026.06.06 /riscv/gcc + ( cd /riscv/gcc; + ./configure --prefix=/opt/riscv-ilp32 --with-arch=rv32gc --with-abi=ilp32 --disable-gdb; + make "$MAKEJOBS"; ) + rm -rf /riscv/gcc +fi + +if [[ "${RUN_IWYU}" == true ]]; then + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${IWYU_LLVM_V}" /include-what-you-use + pushd /include-what-you-use + patch -p1 < /ci_container_base/ci/test/01_iwyu.patch + patch -p1 < /ci_container_base/ci/test/02_iwyu_hash.patch + popd + cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${IWYU_LLVM_V}" -S /include-what-you-use make -C /iwyu-build/ install "$MAKEJOBS" fi @@ -106,7 +110,7 @@ mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar" OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" @@ -114,4 +118,35 @@ if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ] tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi +FREEBSD_SDK_BASENAME="freebsd-${HOST}-${FREEBSD_VERSION}" + +if [ -n "$FREEBSD_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${FREEBSD_SDK_BASENAME}" ]; then + FREEBSD_SDK_FILENAME="base-${FREEBSD_VERSION}.txz" + FREEBSD_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${FREEBSD_SDK_FILENAME}" + if [ ! -f "$FREEBSD_SDK_PATH" ]; then + ${CI_RETRY_EXE} curl --location --fail "https://download.freebsd.org/releases/amd64/${FREEBSD_VERSION}-RELEASE/base.txz" -o "$FREEBSD_SDK_PATH" + fi + mkdir -p "${DEPENDS_DIR}/SDKs/${FREEBSD_SDK_BASENAME}" + tar -C "${DEPENDS_DIR}/SDKs/${FREEBSD_SDK_BASENAME}" -xf "$FREEBSD_SDK_PATH" +fi + +if [ -n "$OPENBSD_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OPENBSD_SDK_BASENAME}" ]; then + mkdir -p "${DEPENDS_DIR}/SDKs/${OPENBSD_SDK_BASENAME}" + for OPENBSD_SDK_FILENAME in base79.tgz comp79.tgz; do + OPENBSD_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OPENBSD_SDK_FILENAME}" + if [ ! -f "$OPENBSD_SDK_PATH" ]; then + ${CI_RETRY_EXE} curl --location --fail "https://cdn.openbsd.org/pub/OpenBSD/${OPENBSD_VERSION}/amd64/${OPENBSD_SDK_FILENAME}" -o "$OPENBSD_SDK_PATH" + fi + tar -C "${DEPENDS_DIR}/SDKs/${OPENBSD_SDK_BASENAME}" -xf "$OPENBSD_SDK_PATH" + ( + # The SDK has versioned shared libs, but no unversioned libfoo.so symlink, + # which breaks linking the kernel with lld. Create the symlinks. + cd "${DEPENDS_DIR}/SDKs/${OPENBSD_SDK_BASENAME}/usr/lib" + ln -sf libc++abi.so.*.* libc++abi.so + ln -sf libc++.so.*.* libc++.so + ln -sf libpthread.so.*.* libpthread.so + ) + done +fi + echo -n "done" > "${CFG_DONE}" diff --git a/ci/test/01_iwyu.patch b/ci/test/01_iwyu.patch new file mode 100644 index 000000000000..794bb74f5875 --- /dev/null +++ b/ci/test/01_iwyu.patch @@ -0,0 +1,634 @@ +Prefer angled brackets over quotes for include directives. +See: https://en.cppreference.com/w/cpp/preprocessor/include.html. + +--- a/iwyu_path_util.cc ++++ b/iwyu_path_util.cc +@@ -222,7 +222,7 @@ bool IsQuotedInclude(StringRef s) { + } + + string AddQuotes(string include_name, bool angled) { +- if (angled) { ++ if (true) { + return "<" + include_name + ">"; + } + return "\"" + include_name + "\""; + + +Prefer C++ headers over C counterparts. +See: https://github.com/include-what-you-use/include-what-you-use/blob/clang_22/iwyu_include_picker.cc#L645-L687. + +--- a/iwyu_include_picker.cc ++++ b/iwyu_include_picker.cc +@@ -104,20 +104,20 @@ const IncludeMapEntry libc_symbol_map[] = { + // equal. The visibility on the symbol-name is ignored; by convention + // we always set it to kPrivate. + { "_POSIX_VDISABLE", kPrivate, "", kPublic }, +- { "abort", kPrivate, "", kPublic }, ++ { "abort", kPrivate, "", kPrivate }, + { "aiocb", kPrivate, "", kPublic }, + { "blkcnt_t", kPrivate, "", kPublic }, + { "blksize_t", kPrivate, "", kPublic }, + { "cc_t", kPrivate, "", kPublic }, +- { "clock_t", kPrivate, "", kPublic }, ++ { "clock_t", kPrivate, "", kPrivate }, + { "clock_t", kPrivate, "", kPublic }, + { "clockid_t", kPrivate, "", kPublic }, +- { "ctermid", kPrivate, "", kPublic }, ++ { "ctermid", kPrivate, "", kPrivate }, + { "daddr_t", kPrivate, "", kPublic }, + { "dev_t", kPrivate, "", kPublic }, +- { "div_t", kPrivate, "", kPublic }, +- { "double_t", kPrivate, "", kPublic }, +- { "error_t", kPrivate, "", kPublic }, ++ { "div_t", kPrivate, "", kPrivate }, ++ { "double_t", kPrivate, "", kPrivate }, ++ { "error_t", kPrivate, "", kPrivate }, + { "error_t", kPrivate, "", kPublic }, + { "error_t", kPrivate, "", kPublic }, + { "FD_CLR", kPrivate, "", kPublic }, +@@ -126,10 +126,10 @@ const IncludeMapEntry libc_symbol_map[] = { + { "fd_set", kPrivate, "", kPublic }, + { "FD_SETSIZE", kPrivate, "", kPublic }, + { "FD_ZERO", kPrivate, "", kPublic }, +- { "fenv_t", kPrivate, "", kPublic }, +- { "fexcept_t", kPrivate, "", kPublic }, +- { "FILE", kPrivate, "", kPublic }, +- { "float_t", kPrivate, "", kPublic }, ++ { "fenv_t", kPrivate, "", kPrivate }, ++ { "fexcept_t", kPrivate, "", kPrivate }, ++ { "FILE", kPrivate, "", kPrivate }, ++ { "float_t", kPrivate, "", kPrivate }, + { "fsblkcnt_t", kPrivate, "", kPublic }, + { "fsfilcnt_t", kPrivate, "", kPublic }, + { "getopt", kPrivate, "", kPublic }, +@@ -139,47 +139,47 @@ const IncludeMapEntry libc_symbol_map[] = { + { "in_addr_t", kPrivate, "", kPublic }, + { "in_port_t", kPrivate, "", kPublic }, + { "id_t", kPrivate, "", kPublic }, +- { "imaxdiv_t", kPrivate, "", kPublic }, +- { "intmax_t", kPrivate, "", kPublic }, +- { "uintmax_t", kPrivate, "", kPublic }, ++ { "imaxdiv_t", kPrivate, "", kPrivate }, ++ { "intmax_t", kPrivate, "", kPrivate }, ++ { "uintmax_t", kPrivate, "", kPrivate }, + { "ino64_t", kPrivate, "", kPublic }, + { "ino_t", kPrivate, "", kPublic }, +- { "int8_t", kPrivate, "", kPublic }, +- { "int16_t", kPrivate, "", kPublic }, +- { "int32_t", kPrivate, "", kPublic }, +- { "int64_t", kPrivate, "", kPublic }, +- { "int_fast8_t", kPrivate, "", kPublic }, +- { "int_fast16_t", kPrivate, "", kPublic }, +- { "int_fast32_t", kPrivate, "", kPublic }, +- { "int_fast64_t", kPrivate, "", kPublic }, +- { "int_least8_t", kPrivate, "", kPublic }, +- { "int_least16_t", kPrivate, "", kPublic }, +- { "int_least32_t", kPrivate, "", kPublic }, +- { "int_least64_t", kPrivate, "", kPublic }, +- { "uint8_t", kPrivate, "", kPublic }, +- { "uint16_t", kPrivate, "", kPublic }, +- { "uint32_t", kPrivate, "", kPublic }, +- { "uint64_t", kPrivate, "", kPublic }, +- { "uint_fast8_t", kPrivate, "", kPublic }, +- { "uint_fast16_t", kPrivate, "", kPublic }, +- { "uint_fast32_t", kPrivate, "", kPublic }, +- { "uint_fast64_t", kPrivate, "", kPublic }, +- { "uint_least8_t", kPrivate, "", kPublic }, +- { "uint_least16_t", kPrivate, "", kPublic }, +- { "uint_least32_t", kPrivate, "", kPublic }, +- { "uint_least64_t", kPrivate, "", kPublic }, +- { "intptr_t", kPrivate, "", kPublic }, +- { "uintptr_t", kPrivate, "", kPublic }, ++ { "int8_t", kPrivate, "", kPrivate }, ++ { "int16_t", kPrivate, "", kPrivate }, ++ { "int32_t", kPrivate, "", kPrivate }, ++ { "int64_t", kPrivate, "", kPrivate }, ++ { "int_fast8_t", kPrivate, "", kPrivate }, ++ { "int_fast16_t", kPrivate, "", kPrivate }, ++ { "int_fast32_t", kPrivate, "", kPrivate }, ++ { "int_fast64_t", kPrivate, "", kPrivate }, ++ { "int_least8_t", kPrivate, "", kPrivate }, ++ { "int_least16_t", kPrivate, "", kPrivate }, ++ { "int_least32_t", kPrivate, "", kPrivate }, ++ { "int_least64_t", kPrivate, "", kPrivate }, ++ { "uint8_t", kPrivate, "", kPrivate }, ++ { "uint16_t", kPrivate, "", kPrivate }, ++ { "uint32_t", kPrivate, "", kPrivate }, ++ { "uint64_t", kPrivate, "", kPrivate }, ++ { "uint_fast8_t", kPrivate, "", kPrivate }, ++ { "uint_fast16_t", kPrivate, "", kPrivate }, ++ { "uint_fast32_t", kPrivate, "", kPrivate }, ++ { "uint_fast64_t", kPrivate, "", kPrivate }, ++ { "uint_least8_t", kPrivate, "", kPrivate }, ++ { "uint_least16_t", kPrivate, "", kPrivate }, ++ { "uint_least32_t", kPrivate, "", kPrivate }, ++ { "uint_least64_t", kPrivate, "", kPrivate }, ++ { "intptr_t", kPrivate, "", kPrivate }, ++ { "uintptr_t", kPrivate, "", kPrivate }, + { "iovec", kPrivate, "", kPublic }, +- { "itimerspec", kPrivate, "", kPublic }, ++ { "itimerspec", kPrivate, "", kPrivate }, + { "key_t", kPrivate, "", kPublic }, +- { "L_ctermid", kPrivate, "", kPublic }, +- { "lconv", kPrivate, "", kPublic }, +- { "ldiv_t", kPrivate, "", kPublic }, +- { "lldiv_t", kPrivate, "", kPublic }, +- { "locale_t", kPrivate, "", kPublic }, +- { "max_align_t", kPrivate, "", kPublic }, +- { "mbstate_t", kPrivate, "", kPublic }, ++ { "L_ctermid", kPrivate, "", kPrivate }, ++ { "lconv", kPrivate, "", kPrivate }, ++ { "ldiv_t", kPrivate, "", kPrivate }, ++ { "lldiv_t", kPrivate, "", kPrivate }, ++ { "locale_t", kPrivate, "", kPrivate }, ++ { "max_align_t", kPrivate, "", kPrivate }, ++ { "mbstate_t", kPrivate, "", kPrivate }, + { "mcontext_t", kPrivate, "", kPublic }, + { "mode_t", kPrivate, "", kPublic }, + { "nl_item", kPrivate, "", kPublic }, +@@ -195,8 +195,8 @@ const IncludeMapEntry libc_symbol_map[] = { + { "optind", kPrivate, "", kPublic }, + { "optopt", kPrivate, "", kPublic }, + { "pid_t", kPrivate, "", kPublic }, +- { "posix_memalign", kPrivate, "", kPublic }, +- { "printf", kPrivate, "", kPublic }, ++ { "posix_memalign", kPrivate, "", kPrivate }, ++ { "printf", kPrivate, "", kPrivate }, + { "pthread_attr_t", kPrivate, "", kPublic }, + { "pthread_cond_t", kPrivate, "", kPublic }, + { "pthread_condattr_t", kPrivate, "", kPublic }, +@@ -207,7 +207,7 @@ const IncludeMapEntry libc_symbol_map[] = { + { "pthread_rwlock_t", kPrivate, "", kPublic }, + { "pthread_rwlockattr_t", kPrivate, "", kPublic }, + { "pthread_t", kPrivate, "", kPublic }, +- { "ptrdiff_t", kPrivate, "", kPublic }, ++ { "ptrdiff_t", kPrivate, "", kPrivate }, + { "regex_t", kPrivate, "", kPublic }, + { "regmatch_t", kPrivate, "", kPublic }, + { "regoff_t", kPrivate, "", kPublic }, +@@ -238,54 +238,54 @@ const IncludeMapEntry libc_symbol_map[] = { + { "SCHED_FIFO", kPrivate, "", kPublic }, + { "SCHED_OTHER", kPrivate, "", kPublic }, + { "SCHED_RR", kPrivate, "", kPublic }, +- { "SEEK_CUR", kPrivate, "", kPublic }, +- { "SEEK_END", kPrivate, "", kPublic }, +- { "SEEK_SET", kPrivate, "", kPublic }, +- { "sig_atomic_t", kPrivate, "", kPublic }, +- { "sigevent", kPrivate, "", kPublic }, +- { "siginfo_t", kPrivate, "", kPublic }, +- { "sigset_t", kPrivate, "", kPublic }, +- { "sigval", kPrivate, "", kPublic }, ++ { "SEEK_CUR", kPrivate, "", kPrivate }, ++ { "SEEK_END", kPrivate, "", kPrivate }, ++ { "SEEK_SET", kPrivate, "", kPrivate }, ++ { "sig_atomic_t", kPrivate, "", kPrivate }, ++ { "sigevent", kPrivate, "", kPrivate }, ++ { "siginfo_t", kPrivate, "", kPrivate }, ++ { "sigset_t", kPrivate, "", kPrivate }, ++ { "sigval", kPrivate, "", kPrivate }, + { "sockaddr", kPrivate, "", kPublic }, + { "socklen_t", kPrivate, "", kPublic }, + { "ssize_t", kPrivate, "", kPublic }, +- { "stack_t", kPrivate, "", kPublic }, ++ { "stack_t", kPrivate, "", kPrivate }, + { "stat", kPrivate, "", kPublic }, + { "suseconds_t", kPrivate, "", kPublic }, +- { "time_t", kPrivate, "", kPublic }, ++ { "time_t", kPrivate, "", kPrivate }, + { "time_t", kPrivate, "", kPublic }, + { "timer_t", kPrivate, "", kPublic }, +- { "timespec", kPrivate, "", kPublic }, ++ { "timespec", kPrivate, "", kPrivate }, + { "timeval", kPrivate, "", kPublic }, // 'canonical' location for timeval + { "timeval", kPrivate, "", kPublic }, + { "timeval", kPrivate, "", kPublic }, + { "timeval", kPrivate, "", kPublic }, +- { "tm", kPrivate, "", kPublic }, ++ { "tm", kPrivate, "", kPrivate }, + { "u_char", kPrivate, "", kPublic }, + { "ucontext_t", kPrivate, "", kPublic }, + { "uid_t", kPrivate, "", kPublic }, + { "useconds_t", kPrivate, "", kPublic }, +- { "wchar_t", kPrivate, "", kPublic }, +- { "wctrans_t", kPrivate, "", kPublic }, +- { "wctype_t", kPrivate, "", kPublic }, ++ { "wchar_t", kPrivate, "", kPrivate }, ++ { "wctrans_t", kPrivate, "", kPrivate }, ++ { "wctype_t", kPrivate, "", kPrivate }, + { "winsize", kPrivate, "", kPublic }, +- { "wint_t", kPrivate, "", kPublic }, ++ { "wint_t", kPrivate, "", kPrivate }, + // It is unspecified if the cname headers provide ::size_t. + // is the one header which defines NULL but not size_t. +- { "size_t", kPrivate, "", kPublic }, // 'canonical' location for size_t +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, +- { "size_t", kPrivate, "", kPublic }, ++ { "size_t", kPrivate, "", kPrivate }, // 'canonical' location for size_t ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, ++ { "size_t", kPrivate, "", kPrivate }, + // Macros that can be defined in more than one file, don't have the + // same __foo_defined guard that other types do, so the grep above + // doesn't discover them. Until I figure out a better way, I just + // add them in by hand as I discover them. +- { "EOF", kPrivate, "", kPublic }, +- { "FILE", kPrivate, "", kPublic }, ++ { "EOF", kPrivate, "", kPrivate }, ++ { "FILE", kPrivate, "", kPrivate }, + { "IBSHIFT", kPrivate, "", kPublic }, + { "MAP_POPULATE", kPrivate, "", kPublic }, + { "MAP_POPULATE", kPrivate, "", kPublic }, +@@ -293,22 +293,22 @@ const IncludeMapEntry libc_symbol_map[] = { + { "MAP_STACK", kPrivate, "", kPublic }, + { "MAXHOSTNAMELEN", kPrivate, "", kPublic }, + { "MAXHOSTNAMELEN", kPrivate, "", kPublic }, +- { "SIGABRT", kPrivate, "", kPublic }, +- { "SIGCHLD", kPrivate, "", kPublic }, +- { "va_arg", kPrivate, "", kPublic }, +- { "va_copy", kPrivate, "", kPublic }, +- { "va_end", kPrivate, "", kPublic }, +- { "va_list", kPrivate, "", kPublic }, +- { "va_start", kPrivate, "", kPublic }, +- { "WEOF", kPrivate, "", kPublic }, ++ { "SIGABRT", kPrivate, "", kPrivate }, ++ { "SIGCHLD", kPrivate, "", kPrivate }, ++ { "va_arg", kPrivate, "", kPrivate }, ++ { "va_copy", kPrivate, "", kPrivate }, ++ { "va_end", kPrivate, "", kPrivate }, ++ { "va_list", kPrivate, "", kPrivate }, ++ { "va_start", kPrivate, "", kPrivate }, ++ { "WEOF", kPrivate, "", kPrivate }, + // These are symbols that could be defined in either stdlib.h or + // malloc.h, but we always want the stdlib location. +- { "malloc", kPrivate, "", kPublic }, +- { "calloc", kPrivate, "", kPublic }, +- { "realloc", kPrivate, "", kPublic }, +- { "free", kPrivate, "", kPublic }, ++ { "malloc", kPrivate, "", kPrivate }, ++ { "calloc", kPrivate, "", kPrivate }, ++ { "realloc", kPrivate, "", kPrivate }, ++ { "free", kPrivate, "", kPrivate }, + // Entries for NULL +- { "NULL", kPrivate, "", kPublic }, // 'canonical' location for NULL ++ { "NULL", kPrivate, "", kPrivate }, // 'canonical' location for NULL + { "NULL", kPrivate, "", kPublic }, + { "NULL", kPrivate, "", kPublic }, + { "NULL", kPrivate, "", kPublic }, +@@ -316,13 +316,13 @@ const IncludeMapEntry libc_symbol_map[] = { + { "NULL", kPrivate, "", kPublic }, + { "NULL", kPrivate, "", kPublic }, + { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "NULL", kPrivate, "", kPublic }, +- { "offsetof", kPrivate, "", kPublic }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "NULL", kPrivate, "", kPrivate }, ++ { "offsetof", kPrivate, "", kPrivate }, + // Auxiliary vector entry types. + { "AT_BASE", kPrivate, "", kPublic }, + { "AT_BASE_PLATFORM", kPrivate, "", kPublic }, +@@ -409,7 +409,7 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -417,18 +417,18 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -436,24 +436,24 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPrivate }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -463,17 +463,17 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -483,22 +483,22 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPrivate }, +@@ -515,12 +515,12 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -532,11 +532,11 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + // Sometimes libc tells you what mapping to do via an '#error': + // # error "Never use directly; include instead." +@@ -546,7 +546,7 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -556,38 +556,38 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + // Top-level #includes that just forward to another file: +@@ -599,13 +599,13 @@ const IncludeMapEntry libc_include_map[] = { + // on the POSIX.1-2024 list, I just choose the top-level one. + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +@@ -625,21 +625,21 @@ const IncludeMapEntry libc_include_map[] = { + { "", kPrivate, "", kPrivate }, + // I don't know what grep would have found these. I found them + // via user report. +- { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, // PATH_MAX ++ { "", kPrivate, "", kPrivate }, // PATH_MAX + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + // System headers available on AIX, BSD, Solaris and other Unix systems + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, +- { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPrivate }, + { "", kPrivate, "", kPublic }, + { "", kPrivate, "", kPublic }, + // Exports guaranteed by the C standard +- { "", kPublic, "", kPublic }, ++ { "", kPublic, "", kPublic }, + }; + + const IncludeMapEntry stdlib_c_include_map[] = { +@@ -658,32 +658,32 @@ const IncludeMapEntry stdlib_c_include_map[] = { + // https://github.com/cplusplus/draft/blob/c+%2B20/source/lib-intro.tex + // + // $ curl -s -N https://raw.githubusercontent.com/cplusplus/draft/c%2B%2B20/source/lib-intro.tex | sed -n '/begin{multicolfloattable}.*{headers.cpp.c}/,/end{multicolfloattable}/p' | grep tcode | perl -nle 'm/tcode{}/ && print qq@ { "<$1.h>", kPublic, "", kPublic },@' | sort +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, + { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, +- { "", kPublic, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, ++ { "", kPrivate, "", kPublic }, + }; + + const char* stdlib_cpp_public_headers[] = { diff --git a/ci/test/02_iwyu_hash.patch b/ci/test/02_iwyu_hash.patch new file mode 100644 index 000000000000..12df7a04ed5c --- /dev/null +++ b/ci/test/02_iwyu_hash.patch @@ -0,0 +1,44 @@ +Map std::hash to its providing standard headers. +Backport of https://github.com/include-what-you-use/include-what-you-use/pull/2013 +(commit 52f85e1f4d990f55fc6556d543eb051d79364a16) to the clang_22 release +branch. Drop once the upstream fix lands in the IWYU branch tracked by +ci/test/00_setup_env_native_iwyu.sh. + +--- a/std_symbol_map.inc ++++ b/std_symbol_map.inc +@@ -1054,12 +1054,27 @@ + { "std::has_unique_object_representations_v", kPrivate, "", kPublic }, + { "std::has_virtual_destructor", kPrivate, "", kPublic }, + { "std::has_virtual_destructor_v", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, + { "std::hash, :0>>", kPrivate, "", kPublic }, ++{ "std::hash>", kPrivate, "", kPublic }, + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, +@@ -1069,6 +1084,7 @@ + { "std::hash>", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, ++{ "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, + { "std::hash", kPrivate, "", kPublic }, diff --git a/ci/test/02_run_container.py b/ci/test/02_run_container.py index 513ecacaca87..05887000a2ba 100755 --- a/ci/test/02_run_container.py +++ b/ci/test/02_run_container.py @@ -3,18 +3,21 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +from pathlib import Path import os import shlex import subprocess import sys +import time def run(cmd, **kwargs): print("+ " + shlex.join(cmd), flush=True) + kwargs.setdefault("check", True) try: - return subprocess.run(cmd, check=True, **kwargs) + return subprocess.run(cmd, **kwargs) except Exception as e: - sys.exit(e) + sys.exit(str(e)) def main(): @@ -23,7 +26,6 @@ def main(): ["bash", "-c", "grep export ./ci/test/00_setup_env*.sh"], stdout=subprocess.PIPE, text=True, - encoding="utf8", ).stdout.splitlines() settings = set(l.split("=")[0].split("export ")[1] for l in settings) # Add "hidden" settings, which are never exported, manually. Otherwise, @@ -36,16 +38,156 @@ def main(): # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. env_file = "/tmp/env-{u}-{c}".format( - u=os.getenv("USER"), - c=os.getenv("CONTAINER_NAME"), + u=os.environ["USER"], + c=os.environ["CONTAINER_NAME"], ) - with open(env_file, "w", encoding="utf8") as file: + with open(env_file, "w") as file: for k, v in os.environ.items(): if k in settings: file.write(f"{k}={v}\n") run(["cat", env_file]) - run(["./ci/test/02_run_container.sh"]) # run the remainder + if os.getenv("DANGER_RUN_CI_ON_HOST"): + print("Running on host system without docker wrapper") + print("Create missing folders") + for create_dir in [ + os.environ["CCACHE_DIR"], + os.environ["PREVIOUS_RELEASES_DIR"], + ]: + Path(create_dir).mkdir(parents=True, exist_ok=True) + + # Modify PATH to prepend the retry script, needed for CI_RETRY_EXE + os.environ["PATH"] = f"{os.environ['BASE_ROOT_DIR']}/ci/retry:{os.environ['PATH']}" + else: + CI_IMAGE_LABEL = "bitcoin-ci-test" + + # Use buildx unconditionally + # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly + cmd_build = ["docker", "buildx", "build"] + cmd_build += [ + f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile", + f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}", + f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}", + f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}", + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + f"--label={CI_IMAGE_LABEL}", + f"--tag={os.environ['CONTAINER_NAME']}", + ] + cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", "")) + cmd_build += [os.environ["BASE_READ_ONLY_DIR"]] + + print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in") + if run(cmd_build, check=False).returncode != 0: + print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure") + time.sleep(3) + run(cmd_build) + + for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]: + run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False) + + CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [] + + if os.getenv("DANGER_CI_ON_HOST_FOLDERS"): + # ensure the directories exist + for create_dir in [ + os.environ["CCACHE_DIR"], + f"{os.environ['DEPENDS_DIR']}/built", + f"{os.environ['DEPENDS_DIR']}/sources", + os.environ["PREVIOUS_RELEASES_DIR"], + os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally + ]: + Path(create_dir).mkdir(parents=True, exist_ok=True) + + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built" + CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources" + CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}" + CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"] + + if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"): + if not os.path.isdir(os.environ["CCACHE_DIR"]): + print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.") + sys.exit(1) + CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" + + run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False) + run(["docker", "network", "create", "--subnet", "1.1.1.0/24", "ci-ip4net"], check=False) + + if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"): + print("Restart docker before run to stop and clear all containers started with --rm") + run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker" + + # Still prune everything in case the filtered pruning doesn't work, or if labels were not set + # on a previous run. Belt and suspenders approach, should be fine to remove in the future. + # Prune images used by --external containers (e.g. build containers) when + # using podman. + print("Prune all dangling images") + run(["podman", "image", "prune", "--force", "--external"]) + + print(f"Prune all dangling {CI_IMAGE_LABEL} images") + # When detecting podman-docker, `--external` should be added. + run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"]) + + cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"] + cmd_run += [ + "--cap-add=LINUX_IMMUTABLE", + *shlex.split(os.getenv("CI_CONTAINER_CAP", "")), + f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly", + f"--mount={CI_CCACHE_MOUNT}", + f"--mount={CI_DEPENDS_MOUNT}", + f"--mount={CI_DEPENDS_SOURCES_MOUNT}", + f"--mount={CI_PREVIOUS_RELEASES_MOUNT}", + *CI_BUILD_MOUNT, + f"--env-file={env_file}", + f"--name={os.environ['CONTAINER_NAME']}", + "--network=ci-ip6net", + "--ip6=1111:1111::5", # Used by some of the tests, don't change it just here (keep them in sync). + f"--platform={os.environ['CI_IMAGE_PLATFORM']}", + os.environ["CONTAINER_NAME"], + ] + + container_id = run( + cmd_run, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + run(["docker", "network", "connect", "--ip=1.1.1.5", "ci-ip4net", container_id]) # The IP address is used by some of the tests, don't change it just here (keep them in sync). + + def ci_exec(cmd_inner, **kwargs): + if os.getenv("DANGER_RUN_CI_ON_HOST"): + prefix = [] + else: + prefix = [ + "docker", + "exec", + "--env", + "DANGER_RUN_CI_ON_HOST=1", # Safe to set *inside* the container + container_id, + ] + + return run([*prefix, *cmd_inner], **kwargs) + + # Normalize all folders to BASE_ROOT_DIR + ci_exec([ + "rsync", + "--recursive", + "--perms", + "--stats", + "--human-readable", + f"{os.environ['BASE_READ_ONLY_DIR']}/", + f"{os.environ['BASE_ROOT_DIR']}", + ]) + ci_exec([f"{os.environ['BASE_ROOT_DIR']}/ci/test/01_base_install.sh"]) + ci_exec([f"{os.environ['BASE_ROOT_DIR']}/ci/test/03_test_script.sh"]) + + if not os.getenv("DANGER_RUN_CI_ON_HOST"): + print("Stop and remove CI container by ID") + run(["docker", "container", "kill", container_id]) if __name__ == "__main__": diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh deleted file mode 100755 index e91f87534dd8..000000000000 --- a/ci/test/02_run_container.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 -export CI_IMAGE_LABEL="bitcoin-ci-test" - -set -o errexit -o pipefail -o xtrace - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - # Env vars during the build can not be changed. For example, a modified - # $MAKEJOBS is ignored in the build process. Use --cpuset-cpus as an - # approximation to respect $MAKEJOBS somewhat, if cpuset is available. - MAYBE_CPUSET="" - if [ "$HAVE_CGROUP_CPUSET" ]; then - MAYBE_CPUSET="--cpuset-cpus=$( python3 -c "import random;P=$( nproc );M=min(P,int('$MAKEJOBS'.lstrip('-j')));print(','.join(map(str,sorted(random.sample(range(P),M)))))" )" - fi - echo "Creating $CI_IMAGE_NAME_TAG container to run in" - - # Use buildx unconditionally - # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly - # shellcheck disable=SC2086 - docker buildx build \ - --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \ - --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ - --build-arg "FILE_ENV=${FILE_ENV}" \ - --build-arg "BASE_ROOT_DIR=${BASE_ROOT_DIR}" \ - $MAYBE_CPUSET \ - --platform="${CI_IMAGE_PLATFORM}" \ - --label="${CI_IMAGE_LABEL}" \ - --tag="${CONTAINER_NAME}" \ - $DOCKER_BUILD_CACHE_ARG \ - "${BASE_READ_ONLY_DIR}" - - docker volume create "${CONTAINER_NAME}_ccache" || true - docker volume create "${CONTAINER_NAME}_depends" || true - docker volume create "${CONTAINER_NAME}_depends_sources" || true - docker volume create "${CONTAINER_NAME}_previous_releases" || true - - CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="" - - if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then - # ensure the directories exist - mkdir -p "${CCACHE_DIR}" - mkdir -p "${DEPENDS_DIR}/built" - mkdir -p "${DEPENDS_DIR}/sources" - mkdir -p "${PREVIOUS_RELEASES_DIR}" - mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally - - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR" - CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built" - CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources" - CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" - CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}" - fi - - if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then - if [ ! -d "${CCACHE_DIR}" ]; then - echo "Error: Directory '${CCACHE_DIR}' must be created in advance." - exit 1 - fi - CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" - fi - - docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true - - if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then - echo "Restart docker before run to stop and clear all containers started with --rm" - podman container rm --force --all # Similar to "systemctl restart docker" - - # Still prune everything in case the filtered pruning doesn't work, or if labels were not set - # on a previous run. Belt and suspenders approach, should be fine to remove in the future. - # Prune images used by --external containers (e.g. build containers) when - # using podman. - echo "Prune all dangling images" - podman image prune --force --external - fi - echo "Prune all dangling $CI_IMAGE_LABEL images" - # When detecting podman-docker, `--external` should be added. - docker image prune --force --filter "label=$CI_IMAGE_LABEL" - - # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ - --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ - --mount "${CI_CCACHE_MOUNT}" \ - --mount "${CI_DEPENDS_MOUNT}" \ - --mount "${CI_DEPENDS_SOURCES_MOUNT}" \ - --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \ - ${CI_BUILD_MOUNT} \ - --env-file /tmp/env-$USER-$CONTAINER_NAME \ - --name "$CONTAINER_NAME" \ - --network ci-ip6net \ - --platform="${CI_IMAGE_PLATFORM}" \ - "$CONTAINER_NAME") - export CI_CONTAINER_ID - export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" -else - echo "Running on host system without docker wrapper" - echo "Create missing folders" - mkdir -p "${CCACHE_DIR}" - mkdir -p "${PREVIOUS_RELEASES_DIR}" -fi - -if [ "$CI_OS_NAME" == "macos" ]; then - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" - export IN_GETOPT_BIN -fi - -CI_EXEC () { - $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*" -} -export -f CI_EXEC - -# Normalize all folders to BASE_ROOT_DIR -CI_EXEC rsync --recursive --perms --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/" -CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" - -CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" - -CI_EXEC "${BASE_ROOT_DIR}/ci/test/03_test_script.sh" - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Stop and remove CI container by ID" - docker container kill "${CI_CONTAINER_ID}" -fi diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 05e4d8fd54a9..7650ab7d86b6 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -6,8 +6,16 @@ export LC_ALL=C.UTF-8 -set -ex +set -o errexit -o xtrace -o pipefail +if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then + echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1" + exit 1 +fi + +cd "${BASE_ROOT_DIR}" + +export PATH="/path_with space:${PATH}" export ASAN_OPTIONS="detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1" @@ -41,28 +49,18 @@ echo "=== BEGIN env ===" env echo "=== END env ===" -( - # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. - # The statistic bytes_written is only used for logging, which is disabled in - # CI, so as a temporary minimal fix to work around UB and CI failures, leave - # bytes_written unmodified. - # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748 - # Tee patch to stdout to make it clear CI is testing modified code. - tee >(patch -p1) <<'EOF' ---- a/src/leveldb/db/db_impl.cc -+++ b/src/leveldb/db/db_impl.cc -@@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { - stats.bytes_read += compact->compaction->input(which, i)->file_size; - } - } -- for (size_t i = 0; i < compact->outputs.size(); i++) { -- stats.bytes_written += compact->outputs[i].file_size; -- } - - mutex_.Lock(); - stats_[compact->compaction->level() + 1].Add(stats); -EOF -) +# The CI framework should be flexible where it is run from. For example, from +# a git-archive, a git-worktree, or a normal git repo. +# The iwyu task requires a working git repo, which may not always be +# available, so initialize one with force. +if [[ "${RUN_IWYU}" == true ]]; then + mv .git .git_ci_backup || true + git init + git add ./src # the git diff command used later for iwyu only cares about ./src + git config user.email "ci@ci" + git config user.name "CI" + git commit -m "dummy CI ./src init for IWYU" +fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/ @@ -82,15 +80,6 @@ elif [ "$RUN_UNIT_TESTS" = "true" ]; then fi fi -if [ "$USE_BUSY_BOX" = "true" ]; then - echo "Setup to use BusyBox utils" - # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) - for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done - # Print BusyBox version - patch --help -fi - # Make sure default datadir does not exist and is never read by creating a dummy file if [ "$CI_OS_NAME" == "macos" ]; then echo > "${HOME}/Library/Application Support/Bitcoin" @@ -99,8 +88,8 @@ else fi if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - SHELL_OPTS="CONFIG_SHELL=/bin/dash" + if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then + SHELL_OPTS="CONFIG_SHELL=/usr/bin/dash" else SHELL_OPTS="CONFIG_SHELL=" fi @@ -110,48 +99,67 @@ if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then test/get_previous_releases.py --target-dir "$PREVIOUS_RELEASES_DIR" fi -BITCOIN_CONFIG_ALL="-DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON" +BITCOIN_CONFIG_ALL="-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON" if [ -z "$NO_DEPENDS" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} -DCMAKE_TOOLCHAIN_FILE=$DEPENDS_DIR/$HOST/toolchain.cmake" fi -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} -DWERROR=ON" -fi ccache --zero-stats -PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" # Folder where the build is done. BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST} -BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR -Werror=dev" +BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL '-DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR' -Werror=dev" -if [[ "${RUN_TIDY}" == "true" ]]; then +if [[ "${RUN_IWYU}" == true || "${RUN_TIDY}" == true ]]; then BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" fi -bash -c "cmake -S $BASE_ROOT_DIR -B ${BASE_BUILD_DIR} $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( +eval "CMAKE_ARGS=($BITCOIN_CONFIG_ALL $BITCOIN_CONFIG)" +cmake -S "$BASE_ROOT_DIR" -B "$BASE_BUILD_DIR" "${CMAKE_ARGS[@]}" || ( cd "${BASE_BUILD_DIR}" # shellcheck disable=SC2046 cat $(cmake -P "${BASE_ROOT_DIR}/ci/test/GetCMakeLogFiles.cmake") false ) +if [[ "${GOAL}" != all && "${GOAL}" != codegen ]]; then + GOAL="all ${GOAL}" +fi + # shellcheck disable=SC2086 -cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target all $GOAL || ( +cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target $GOAL || ( echo "Build failure. Verbose build follows." # shellcheck disable=SC2086 - cmake --build "${BASE_BUILD_DIR}" -j1 --target all $GOAL --verbose + cmake --build "${BASE_BUILD_DIR}" -j1 --target $GOAL --verbose false ) -bash -c "${PRINT_CCACHE_STATISTICS}" -if [ "$CI" = "true" ]; then - hit_rate=$(ccache -s | grep "Hits:" | head -1 | sed 's/.*(\(.*\)%).*/\1/') - if [ "${hit_rate%.*}" -lt 75 ]; then - echo "::notice title=low ccache hitrate::Ccache hit-rate in $CONTAINER_NAME was $hit_rate%" - fi -fi +ccache --version | head -n 1 && ccache --show-stats --verbose +ccache --print-stats | python3 -c ' +import os +import sys + +for line in sys.stdin: + key, value = line.split("\t", 1) + # "primary storage" fallback only needed for ccache version 4.5.1 + if key in ("local_storage_hit", "primary_storage_hit"): + hits = int(value) + elif key in ("local_storage_miss", "primary_storage_miss"): + miss = int(value) + +calls = hits + miss +# codegen has no calls, so skip that here +if calls: + rate = (hits / calls * 100) + print(f"{rate:.2f}") + if rate < 75: + container = os.environ["CONTAINER_NAME"] + print( + "::notice title=low ccache hitrate::" + f"Ccache hit-rate in {container} was {rate:.2f}%" + ) +' du -sh "${DEPENDS_DIR}"/*/ du -sh "${PREVIOUS_RELEASES_DIR}" @@ -159,14 +167,27 @@ if [ -n "${CI_LIMIT_STACK_SIZE}" ]; then ulimit -s 512 fi +if [[ ${BARE_METAL_RISCV} == "true" ]]; then + export BASE_BUILD_DIR + "${BASE_ROOT_DIR}/ci/test/link-riscv.sh" +fi + if [ -n "$USE_VALGRIND" ]; then - "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" + "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.py" fi if [ "$RUN_CHECK_DEPS" = "true" ]; then "${BASE_ROOT_DIR}/contrib/devtools/check-deps.sh" "${BASE_BUILD_DIR}" fi +if [[ "$CI_OS_NAME" == "macos" && "${GOAL}" = "install deploy" ]]; then + unzip "${BASE_BUILD_DIR}/bitcoin-macos-app.zip" -d "${BASE_BUILD_DIR}/deploy" + if ! ( codesign --verify "${BASE_BUILD_DIR}/deploy/Bitcoin-Qt.app" ); then + echo "Codesigning failed." + false + fi +fi + if [ "$RUN_UNIT_TESTS" = "true" ]; then DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" \ LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \ @@ -182,7 +203,7 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then eval "TEST_RUNNER_EXTRA=($TEST_RUNNER_EXTRA)" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \ "${BASE_BUILD_DIR}/test/functional/test_runner.py" \ - --ci "${MAKEJOBS}" \ + "${MAKEJOBS}" \ --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" \ --ansi \ --combinedlogslen=99999999 \ @@ -204,20 +225,42 @@ if [ "${RUN_TIDY}" = "true" ]; then mv tmp.json "${BASE_BUILD_DIR}/compile_commands.json" cd "${BASE_BUILD_DIR}/src/" - if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then + if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -config-file="${BASE_ROOT_DIR}/src/.clang-tidy" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then grep -C5 "error: " tmp.tidy-out.txt echo "^^^ ⚠️ Failure generated from clang-tidy" false fi +fi + +if [[ "${RUN_IWYU}" == true ]]; then + # TODO: Consider enforcing IWYU across the entire codebase. + FILES_WITH_ENFORCED_IWYU="/src/(((bench|crypto|index|kernel|primitives|script|univalue/(lib|test)|util|zmq)/.*|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" + jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" cd "${BASE_ROOT_DIR}" - python3 "/include-what-you-use/iwyu_tool.py" \ - -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ - -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ - -Xiwyu --max_line_length=160 \ - 2>&1 | tee /tmp/iwyu_ci.out - cd "${BASE_ROOT_DIR}/src" - python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + + run_iwyu() { + mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json" + { + python3 "/include-what-you-use/iwyu_tool.py" \ + -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ + -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ + -Xiwyu --check_also="*/primitives/*.h" \ + 2>&1 || true + } | tee /tmp/iwyu_ci.out + python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + git diff -U1 | ./contrib/devtools/clang-format-diff.py -binary="clang-format-${IWYU_LLVM_V}" -p1 -i -v + } + + run_iwyu "compile_commands_iwyu_errors.json" + if ! ( git --no-pager diff --exit-code ); then + echo "^^^ ⚠️ Failure generated from IWYU" + false + fi + + run_iwyu "compile_commands_iwyu_warnings.json" git --no-pager diff fi diff --git a/ci/test/link-riscv.sh b/ci/test/link-riscv.sh new file mode 100755 index 000000000000..93bebfa8a97e --- /dev/null +++ b/ci/test/link-riscv.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 +set -o errexit -o xtrace -o pipefail + +GCC=/opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc +GXX=/opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ +CRTBEGIN=$("${GCC}" -print-file-name=crtbegin.o) +LIBGCC=$("${GCC}" -print-libgcc-file-name) +LIBSTDCXX=$("${GXX}" -print-file-name=libstdc++.a) +LIBC=$("${GCC}" -print-file-name=libc.a) +LIBM=$("${GCC}" -print-file-name=libm.a) + +echo -e "#include