diff --git a/.buildkite/instantiate_and_test.sh b/.buildkite/instantiate_and_test.sh deleted file mode 100644 index 23d0711..0000000 --- a/.buildkite/instantiate_and_test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -julia --project=test -e 'import Pkg; Pkg.instantiate()' -echo '+++ runtests.jl' -julia --project=test test/runtests.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index a60c9a5..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,78 +0,0 @@ -steps: - - label: ":rocket: Launch matrix of options" - commands: | - # These variables are available for templating in our child pipelines - export JULIA_VERSION OS ARCH AGENT_ARCH ROOTFS_URL ROOTFS_HASH DOCKER_IMAGE - - function upload_pipeline() { - buildkite-agent pipeline upload .buildkite/test_$${OS}.yml - } - - # We'll test on the following julia versions - for JULIA_VERSION in 1.10 nightly; do - # First, linux - OS="linux" - - ARCH="x86_64" - AGENT_ARCH="x86_64" - ROOTFS_URL="https://github.com/JuliaCI/rootfs-images/releases/download/v7.5/package_linux.x86_64.tar.gz" - ROOTFS_HASH="c615fcc0a6923d520f11cc17934d4f7ea2ddb78d" - upload_pipeline - - # We still schedule this one on an x86_64 machine, but use an i686 rootfs - ARCH="i686" - AGENT_ARCH="x86_64" - ROOTFS_URL="https://github.com/JuliaCI/rootfs-images/releases/download/v7.5/package_linux.i686.tar.gz" - ROOTFS_HASH="6c24bdecd727cc6027039e2514d1d5a6490a99fd" - upload_pipeline - - ARCH="aarch64" - AGENT_ARCH="aarch64" - ROOTFS_URL="https://github.com/JuliaCI/rootfs-images/releases/download/v7.5/package_linux.aarch64.tar.gz" - ROOTFS_HASH="c4223131e1c45aa676d5feca888668477d6e69a4" - upload_pipeline - - # Next, windows - OS="windows" - ARCH="x86_64" - AGENT_ARCH="x86_64" - DOCKER_IMAGE="juliapackaging/package-windows-x86_64:v7.5" - upload_pipeline - - # same as for Linux, we schedule on an x86_64 machine, but use an i686 docker image - OS="windows" - ARCH="i686" - AGENT_ARCH="x86_64" - DOCKER_IMAGE="juliapackaging/package-windows-i686:v7.5" - upload_pipeline - - - - # Finally, macOS - OS="macos" - for ARCH in x86_64 aarch64; do - AGENT_ARCH="$${ARCH}" - upload_pipeline - done - done - - # We don't have nightly or 1.10 for this one, we have to use Julia v1.7 - JULIA_VERSION="1.7" - ARCH="armv7l" - AGENT_ARCH="armv7l" - ROOTFS_URL="https://github.com/JuliaCI/rootfs-images/releases/download/v7.5/package_linux.armv7l.tar.gz" - ROOTFS_HASH="1fcb42491944e00429fa3cab96692d3795435d50" - OS="linux" - upload_pipeline - agents: - queue: "juliaecosystem" - - - label: "func_list idempotency" - commands: | - cd ext/gensymbol - ./generate_func_list.sh - git diff --exit-code - agents: - queue: "juliaecosystem" - os: "linux" - arch: "x86_64" diff --git a/.buildkite/test_linux.yml b/.buildkite/test_linux.yml deleted file mode 100644 index 18b6bfc..0000000 --- a/.buildkite/test_linux.yml +++ /dev/null @@ -1,31 +0,0 @@ -steps: - - label: ":julia: :linux: ${ARCH?} Julia ${JULIA_VERSION?}" - plugins: - # Install Julia for the agent first, so that we can run `sandbox` - - JuliaCI/julia#v1: - version: "1.7" - arch: "${AGENT_ARCH?}" - - staticfloat/sandbox#v1: - rootfs_url: "${ROOTFS_URL?}" - rootfs_treehash: "${ROOTFS_HASH?}" - uid: 0 - gid: 0 - workspaces: - - "/cache:/cache" - # Then install Julia for the rootfs arch - - JuliaCI/julia#v1: - version: "${JULIA_VERSION?}" - arch: "${ARCH?}" - - staticfloat/metahook#sf/windows_backslashes: - pre-command: | - # Upgrade our debian package to bullseye (wowza) and get an ILP64 blas - if [[ "${ARCH?}" == *64 ]]; then - echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list - apt update && apt install -y libblas64-dev liblapack64-dev - fi - command: "bash .buildkite/instantiate_and_test.sh" - agents: - queue: "juliaecosystem" - os: "linux" - sandbox_capable: "true" - arch: "${AGENT_ARCH?}" diff --git a/.buildkite/test_macos.yml b/.buildkite/test_macos.yml deleted file mode 100644 index 3f96940..0000000 --- a/.buildkite/test_macos.yml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - - label: ":julia: :macos: ${ARCH?} Julia ${JULIA_VERSION?}" - plugins: - - JuliaCI/julia#v1: - version: "${JULIA_VERSION?}" - - staticfloat/metahook#sf/windows_backslashes: - pre-command: | - echo "This is a test" - command: "bash .buildkite/instantiate_and_test.sh" - agents: - queue: "juliaecosystem" - os: "macos" - arch: "${AGENT_ARCH?}" diff --git a/.buildkite/test_windows.yml b/.buildkite/test_windows.yml deleted file mode 100644 index 9e0b1f1..0000000 --- a/.buildkite/test_windows.yml +++ /dev/null @@ -1,22 +0,0 @@ -steps: - - label: ":julia: :windows: ${ARCH?} Julia ${JULIA_VERSION?}" - plugins: - - JuliaCI/julia#v1: - version: "${JULIA_VERSION?}" - arch: "${ARCH?}" - - staticfloat/metahook#sf/windows_backslashes: - # Copy our julia installation to a known location that we can mount with docker - pre-command: | - cp -r $(dirname $(dirname $(which julia)))/* /c/buildkite-agent - - docker#v3.13.0: - image: "${DOCKER_IMAGE?}" - always-pull: true - volumes: - # Abuse the fact that we have a `PATH` mapping for `buildkite-agent` - # to mount in our build of Julia for use within the container: - - "C:\\buildkite-agent:C:\\buildkite-agent" - command: ["bash", "-c", ".buildkite/instantiate_and_test.sh"] - agents: - queue: "juliaecosystem" - os: "windows" - arch: "${AGENT_ARCH?}" diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index 5059197..0000000 --- a/.cirrus.yml +++ /dev/null @@ -1,43 +0,0 @@ -task: - matrix: - - name: FreeBSD - freebsd_instance: - image_family: freebsd-14-2 - env: - matrix: - - JULIA_VERSION: 1 - - name: musl Linux - container: - image: alpine:3.14 - env: - - JULIA_VERSION: 1 - install_script: | - URL="https://raw.githubusercontent.com/ararslan/CirrusCI.jl/master/bin/install.sh" - set -x - if [ "$(uname -s)" = "Linux" ] && command -v apt; then - apt update - apt install -y curl - fi - if command -v curl; then - sh -c "$(curl ${URL})" - elif command -v wget; then - sh -c "$(wget ${URL} -O-)" - elif command -v fetch; then - sh -c "$(fetch ${URL} -o -)" - else - echo "No downloader installed! :( Install one! :)" - fi - - if command -v apk; then - # Install build tools on Alpine Linux - apk update - apk add build-base linux-headers - elif command -v pkg; then - # Install GNU Make on FreeBSD - pkg update - pkg install -y gmake - fi - build_script: - - julia --project=test/ -e 'using Pkg; Pkg.instantiate()' - test_script: - - julia --project=test/ test/runtests.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..25e60d0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,168 @@ +name: CI + +on: + push: + branches: + - main + tags: ['*'] + pull_request: + workflow_dispatch: + +concurrency: + # Cancel in-progress runs for the same PR / branch, but never on `main`. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.backend }} / ${{ matrix.os }} / julia ${{ matrix.julia-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # Each BLAS/LAPACK backend is tested in its own job (and its own process, with + # its own `test/backends//Project.toml`) so that only a single set of + # BLAS/LAPACK libraries is ever loaded at once. + backend: [openblas, mkl, refblas, blis, blas64, accelerate, direct] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + julia-version: ['1.10', 'nightly'] + exclude: + # MKL is x86_64-only. + - {backend: mkl, os: ubuntu-24.04-arm} + - {backend: mkl, os: macos-latest} + # Accelerate is a macOS-only system framework. + - {backend: accelerate, os: ubuntu-latest} + - {backend: accelerate, os: ubuntu-24.04-arm} + - {backend: accelerate, os: windows-latest} + # blas64 uses the Debian `libblas64-dev`/`liblapack64-dev` packages. + - {backend: blas64, os: windows-latest} + - {backend: blas64, os: macos-latest} + - {backend: blas64, os: macos-15-intel} + # blis_jll has no Windows build we exercise here. + - {backend: blis, os: windows-latest} + # Reference BLAS DLLs are not yet resolvable on Windows. + - {backend: refblas, os: windows-latest} + # Only the core `openblas` and `direct` backends run on nightly; every other + # backend runs on the 1.10 LTS only. + - {backend: mkl, julia-version: 'nightly'} + - {backend: refblas, julia-version: 'nightly'} + - {backend: blis, julia-version: 'nightly'} + - {backend: blas64, julia-version: 'nightly'} + - {backend: accelerate, julia-version: 'nightly'} + steps: + - uses: actions/checkout@v6 + + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.julia-version }} + arch: ${{ (matrix.os == 'ubuntu-24.04-arm' || matrix.os == 'macos-latest') && 'aarch64' || 'x64' }} + + - uses: julia-actions/cache@v3 + + # The `blas64` backend tests against a system-provided ILP64 reference BLAS/LAPACK. + - name: Install system libblas64 / liblapack64 (Debian) + if: matrix.backend == 'blas64' + run: | + sudo apt-get update + sudo apt-get install -y libblas64-dev liblapack64-dev + + # Windows needs a MinGW toolchain (gcc + make) to build the C test programs and + # libblastrampoline itself. + - uses: msys2/setup-msys2@v2 + id: msys2 + if: runner.os == 'Windows' + with: + msystem: MINGW64 + path-type: inherit + install: >- + make + mingw-w64-x86_64-gcc + - name: Add MSYS2 tools to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + Add-Content -Path $env:GITHUB_PATH -Value "${{ steps.msys2.outputs.msys2-location }}\mingw64\bin" + Add-Content -Path $env:GITHUB_PATH -Value "${{ steps.msys2.outputs.msys2-location }}\usr\bin" + + - name: Instantiate ${{ matrix.backend }} project + run: julia --project=test/backends/${{ matrix.backend }} -e 'import Pkg; Pkg.instantiate()' + + - name: Run ${{ matrix.backend }} tests + run: julia --project=test/backends/${{ matrix.backend }} test/runtests.jl ${{ matrix.backend }} + + # 32-bit (i686) Linux and Windows: GitHub only offers 64-bit runners, so we install a + # 32-bit Julia on the x86_64 runner (Pkg then resolves i686 JLL artifacts) and build the + # C tests / libblastrampoline as 32-bit (`-m32` on Linux via gcc-multilib; a native i686 + # MinGW toolchain on Windows). + test-32bit: + name: ${{ matrix.backend }} / ${{ matrix.os }} (i686) / julia ${{ matrix.julia-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + backend: [openblas, refblas, direct] + os: [ubuntu-latest, windows-latest] + julia-version: ['1.10', 'nightly'] + exclude: + # Reference BLAS DLLs are not yet resolvable on Windows. + - {backend: refblas, os: windows-latest} + # Only the core `openblas` and `direct` backends run on nightly. + - {backend: refblas, julia-version: 'nightly'} + steps: + - uses: actions/checkout@v6 + + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.julia-version }} + arch: x86 + + - uses: julia-actions/cache@v3 + + # Linux: a 64-bit gcc with multilib can target i686 via `-m32` (utils.jl's + # `needs_m32()` adds it automatically when GCC is 64-bit but Julia is 32-bit). + - name: Install 32-bit build support (Debian) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y gcc-multilib + + # Windows: use a native i686 MinGW toolchain (MINGW32) so the C tests and + # libblastrampoline are built as 32-bit directly. + - uses: msys2/setup-msys2@v2 + id: msys2 + if: runner.os == 'Windows' + with: + msystem: MINGW32 + path-type: inherit + install: >- + make + mingw-w64-i686-gcc + - name: Add MSYS2 tools to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + Add-Content -Path $env:GITHUB_PATH -Value "${{ steps.msys2.outputs.msys2-location }}\mingw32\bin" + Add-Content -Path $env:GITHUB_PATH -Value "${{ steps.msys2.outputs.msys2-location }}\usr\bin" + + - name: Instantiate ${{ matrix.backend }} project + run: julia --project=test/backends/${{ matrix.backend }} -e 'import Pkg; Pkg.instantiate()' + + - name: Run ${{ matrix.backend }} tests + run: julia --project=test/backends/${{ matrix.backend }} test/runtests.jl ${{ matrix.backend }} + + func-list-idempotency: + name: func_list idempotency + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Regenerate func_list and check for diff + run: | + cd ext/gensymbol + ./generate_func_list.sh + git diff --exit-code diff --git a/.gitignore b/.gitignore index c89602e..c1432b9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ src/build/ src/prefix/ test/Manifest.toml +test/backends/*/Manifest.toml diff --git a/test/backends/accelerate/Project.toml b/test/backends/accelerate/Project.toml new file mode 100644 index 0000000..9114ea9 --- /dev/null +++ b/test/backends/accelerate/Project.toml @@ -0,0 +1,2 @@ +# Accelerate is a macOS system framework, so this backend has no JLL dependencies. +[deps] diff --git a/test/accelerate.jl b/test/backends/accelerate/runtests.jl similarity index 56% rename from test/accelerate.jl rename to test/backends/accelerate/runtests.jl index 177b7d4..54b03e1 100644 --- a/test/accelerate.jl +++ b/test/backends/accelerate/runtests.jl @@ -1,14 +1,51 @@ -using Libdl, Test +include(joinpath(@__DIR__, "..", "..", "common.jl")) + +# All of the Accelerate tests are macOS-only, so this whole backend is a no-op elsewhere. +if !Sys.isapple() + @info("Accelerate is only available on macOS; skipping Accelerate tests") +else + +lbt_link_name, lbt_dir = build_libblastrampoline() +lbt_dir = joinpath(lbt_dir, binlib) + +# Do we have Accelerate available? Note that we can't use `isfile()` here, since Apple +# has Helpfully (TM) sequestered the dynamic libraries away into a database somewhere that +# just gets fed into `dlopen()` when you ask for that magic path. +veclib_blas_path = "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/libBLAS.dylib" +if dlopen_e(veclib_blas_path) != C_NULL + # Test that we can run BLAS-only tests without LAPACK loaded (`sgesv` test requires LAPACK symbols) + @testset "LBT -> vecLib/libBLAS" begin + run_all_tests(blastrampoline_link_name(), [lbt_dir], :LP64, veclib_blas_path; tests=[dgemm, sdot, cdotc]) + end + + # With LAPACK as well, run all tests except `dgemmt` + veclib_lapack_path = "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/libLAPACK.dylib" + @testset "LBT -> vecLib/libLAPACK" begin + run_all_tests(blastrampoline_link_name(), [lbt_dir], :LP64, string(veclib_blas_path, ";", veclib_lapack_path)) + end + + veclib_lapack_handle = dlopen(veclib_lapack_path) + if dlsym_e(veclib_lapack_handle, "dpotrf\$NEWLAPACK\$ILP64") != C_NULL + @testset "LBT -> vecLib/libBLAS (ILP64)" begin + veclib_blas_path_ilp64 = "$(veclib_blas_path)!\x1a\$NEWLAPACK\$ILP64" + run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_blas_path_ilp64; tests=[dgemm, sdot, cdotc]) + end + + @testset "LBT -> vecLib/libLAPACK (ILP64)" begin + veclib_lapack_path_ilp64 = "$(veclib_lapack_path)!\x1a\$NEWLAPACK\$ILP64" + @warn("dpstrf test broken on new LAPACK in Accelerate") + dpstrf_broken = (dpstrf[1], "diag(A): 2.2601 1.7140 0.6206 1.1878", true) + run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_lapack_path_ilp64; tests=[dgemm, dpstrf_broken, sgesv, sdot, cdotc]) + end + end +end +# In-process Accelerate tests, driven through the LBT C API directly. # Taken from AppleAccelerate.jl to avoid a dependency on it const libacc = "/System/Library/Frameworks/Accelerate.framework/Accelerate" const libacc_info_plist = "/System/Library/Frameworks/Accelerate.framework/Versions/Current/Resources/Info.plist" function get_macos_version(normalize=true) - @static if !Sys.isapple() - return nothing - end - plist_lines = split(String(read("/System/Library/CoreServices/SystemVersion.plist")), "\n") vers_idx = findfirst(l -> occursin("ProductVersion", l), plist_lines) if vers_idx === nothing @@ -27,9 +64,10 @@ function get_macos_version(normalize=true) return ver end - -# Load the Accelerate library +# Load libblastrampoline and the Accelerate library +lbt_handle = open_lbt_handle() libacc_handle = dlopen(libacc) + @testset "Accelerate ILP64 loading" begin # ILP64 requires macOS 13.3+ if get_macos_version() >= v"13.3" @@ -91,3 +129,5 @@ end @test lbt_get_num_threads(lbt_handle) == 1 end end + +end # if Sys.isapple() diff --git a/test/backends/blas64/Project.toml b/test/backends/blas64/Project.toml new file mode 100644 index 0000000..7f2b38a --- /dev/null +++ b/test/backends/blas64/Project.toml @@ -0,0 +1,3 @@ +# This backend uses a system-provided `libblas64`/`liblapack64` (e.g. the Debian +# `libblas64-dev` / `liblapack64-dev` packages), so it has no JLL dependencies. +[deps] diff --git a/test/backends/blas64/runtests.jl b/test/backends/blas64/runtests.jl new file mode 100644 index 0000000..018afa6 --- /dev/null +++ b/test/backends/blas64/runtests.jl @@ -0,0 +1,25 @@ +include(joinpath(@__DIR__, "..", "..", "common.jl")) + +# Test against a system-provided `libblas64`/`liblapack64` (ILP64), e.g. the Debian +# `libblas64-dev` / `liblapack64-dev` packages. This is gated on those libraries +# actually being installed, so the backend is a no-op when they are missing. +lbt_link_name, lbt_dir = build_libblastrampoline() +lbt_dir = joinpath(lbt_dir, binlib) + +blas64 = dlopen("libblas64", throw_error=false) +if blas64 !== nothing + # Test that we can run BLAS-only tests without LAPACK loaded (`sgesv` test requires LAPACK symbols, blas64 doesn't have CBLAS) + @testset "LBT -> libblas64 (ILP64, BLAS)" begin + run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, dlpath(blas64); tests=[dgemm, sdot]) + end + + # Check if we have a `liblapack` and if we do, run again, this time including `sgesv` + lapack = dlopen("liblapack64", throw_error=false) + if lapack !== nothing + @testset "LBT -> libblas64 + liblapack64 (ILP64, BLAS+LAPACK)" begin + run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, "$(dlpath(blas64));$(dlpath(lapack))"; tests=[dgemm, sdot, sgesv]) + end + end +else + @info("No system `libblas64` found; skipping libblas64 tests") +end diff --git a/test/backends/blis/Project.toml b/test/backends/blis/Project.toml new file mode 100644 index 0000000..1c20e71 --- /dev/null +++ b/test/backends/blis/Project.toml @@ -0,0 +1,3 @@ +[deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +blis_jll = "6136c539-28a5-5bf0-87cc-b183200dce32" diff --git a/test/backends/blis/runtests.jl b/test/backends/blis/runtests.jl new file mode 100644 index 0000000..3286fe7 --- /dev/null +++ b/test/backends/blis/runtests.jl @@ -0,0 +1,21 @@ +include(joinpath(@__DIR__, "..", "..", "common.jl")) +using blis_jll, CompilerSupportLibraries_jll + +# BLIS is an ILP64, BLAS-only library (it ships `dgemm_64_` etc., but no CBLAS and no +# LAPACK), so we only exercise the BLAS routines here. +# +# Pairing BLIS with a separate LAPACK to test `dpstrf`/`sgesv` is blocked on the same +# issue as the ILP64 reference backend: the ILP64 `LAPACK_jll` artifact exports both the +# plain and `_64_`-suffixed symbols, so LBT autodetects it as LP64 and never forwards the +# ILP64 LAPACK symbols. Re-enable a `blis + LAPACK` testset once that is fixed upstream. +if blis_jll.is_available() && Sys.WORD_SIZE == 64 + lbt_link_name, lbt_dir = build_libblastrampoline() + lbt_dir = joinpath(lbt_dir, binlib) + + @testset "LBT -> blis_jll (ILP64, BLAS)" begin + libdirs = unique(vcat(lbt_dir, blis_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, :ILP64, blis_jll.blis_path; tests = [dgemm, sdot]) + end +else + @info("blis_jll is not available on this platform; skipping BLIS tests") +end diff --git a/test/Project.toml b/test/backends/direct/Project.toml similarity index 87% rename from test/Project.toml rename to test/backends/direct/Project.toml index 4e0d3c2..8af9e67 100644 --- a/test/Project.toml +++ b/test/backends/direct/Project.toml @@ -1,5 +1,4 @@ [deps] -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7" OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" diff --git a/test/direct.jl b/test/backends/direct/runtests.jl similarity index 92% rename from test/direct.jl rename to test/backends/direct/runtests.jl index 87e342b..7703c7a 100644 --- a/test/direct.jl +++ b/test/backends/direct/runtests.jl @@ -1,4 +1,6 @@ -using Libdl, Test, OpenBLAS_jll, OpenBLAS32_jll, MKL_jll +include(joinpath(@__DIR__, "..", "..", "common.jl")) +using OpenBLAS_jll, OpenBLAS32_jll, MKL_jll + if isdefined(Base, :pkgversion) openblas32_version = pkgversion(OpenBLAS32_jll) openblas32_version = VersionNumber(openblas32_version.major, openblas32_version.minor, openblas32_version.patch) @@ -7,36 +9,7 @@ if isdefined(Base, :pkgversion) end end -include("utils.jl") - -function unpack_loaded_libraries(config::lbt_config_t) - libs = LBTLibraryInfo[] - idx = 1 - lib_ptr = unsafe_load(config.loaded_libs, idx) - while lib_ptr != C_NULL - push!(libs, LBTLibraryInfo(unsafe_load(lib_ptr), config.num_exported_symbols)) - - idx += 1 - lib_ptr = unsafe_load(config.loaded_libs, idx) - end - return libs -end - -function find_symbol_offset(config::lbt_config_t, symbol::String) - for sym_idx in 1:config.num_exported_symbols - if unsafe_string(unsafe_load(config.exported_symbols, sym_idx)) == symbol - return UInt32(sym_idx - 1) - end - end - return nothing -end - -function bitfield_get(field::Vector{UInt8}, symbol_idx::UInt32) - return field[div(symbol_idx,8)+1] & (UInt8(0x01) << (symbol_idx%8)) -end - -lbt_link_name, lbt_prefix = build_libblastrampoline() -lbt_handle = dlopen("$(lbt_prefix)/$(binlib)/lib$(lbt_link_name).$(shlib_ext)", RTLD_GLOBAL | RTLD_DEEPBIND) +lbt_handle = open_lbt_handle() @testset "Config" begin @test lbt_handle != C_NULL diff --git a/test/backends/mkl/Project.toml b/test/backends/mkl/Project.toml new file mode 100644 index 0000000..fc872e5 --- /dev/null +++ b/test/backends/mkl/Project.toml @@ -0,0 +1,3 @@ +[deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +MKL_jll = "856f044c-d86e-5d09-b602-aeab76dc8ba7" diff --git a/test/backends/mkl/runtests.jl b/test/backends/mkl/runtests.jl new file mode 100644 index 0000000..8cb4d3e --- /dev/null +++ b/test/backends/mkl/runtests.jl @@ -0,0 +1,31 @@ +include(joinpath(@__DIR__, "..", "..", "common.jl")) +using MKL_jll, CompilerSupportLibraries_jll + +# Test against MKL_jll using `libmkl_rt`, which is :LP64 by default +if MKL_jll.is_available() + lbt_link_name, lbt_dir = build_libblastrampoline() + lbt_dir = joinpath(lbt_dir, binlib) + + # On i686, we can't do complex return style autodetection, so we manually set it, + # knowing that MKL is argument-style. + extra_env = Dict{String,String}() + if Sys.ARCH == :i686 + extra_env["LBT_FORCE_RETSTYLE"] = "ARGUMENT" + end + @testset "LBT -> MKL_jll (LP64)" begin + libdirs = unique(vcat(lbt_dir, MKL_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, :LP64, MKL_jll.libmkl_rt_path; tests = [dgemm, dgemmt, dpstrf, sgesv, sdot, cdotc], extra_env) + end + + # Test that we can set MKL's interface via an environment variable to select ILP64, and LBT detects it properly + if Sys.WORD_SIZE == 64 + @testset "LBT -> MKL_jll (ILP64, via env)" begin + withenv("MKL_INTERFACE_LAYER" => "ILP64") do + libdirs = unique(vcat(lbt_dir, MKL_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, :ILP64, MKL_jll.libmkl_rt_path; tests = [dgemm, dgemmt, dpstrf, sgesv, sdot, cdotc], extra_env) + end + end + end +else + @info("MKL_jll is not available on this platform; skipping MKL tests") +end diff --git a/test/backends/openblas/Project.toml b/test/backends/openblas/Project.toml new file mode 100644 index 0000000..4fa0680 --- /dev/null +++ b/test/backends/openblas/Project.toml @@ -0,0 +1,8 @@ +[deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +OpenBLAS32_jll = "656ef2d0-ae68-5445-9ca0-591084a874a2" +OpenBLAS_jll = "4536629a-c528-5b80-bd46-f80d51c5b363" + +# Force an old version of OpenBLAS32_jll so that it's missing some newer symbols +[compat] +OpenBLAS32_jll = "=0.3.10" diff --git a/test/backends/openblas/runtests.jl b/test/backends/openblas/runtests.jl new file mode 100644 index 0000000..f45b0dc --- /dev/null +++ b/test/backends/openblas/runtests.jl @@ -0,0 +1,73 @@ +include(joinpath(@__DIR__, "..", "..", "common.jl")) +using OpenBLAS_jll, OpenBLAS32_jll, CompilerSupportLibraries_jll + +# Build version that links against vanilla OpenBLAS +openblas_interface = :LP64 +if Sys.WORD_SIZE == 64 + openblas_interface = :ILP64 +end +openblas_jll_libname = splitext(basename(OpenBLAS_jll.libopenblas_path)[4:end])[1] +@testset "Vanilla OpenBLAS_jll ($(openblas_interface))" begin + run_all_tests(openblas_jll_libname, OpenBLAS_jll.LIBPATH_list, openblas_interface, "") +end + +# Build version that links against vanilla OpenBLAS32 +@testset "Vanilla OpenBLAS32_jll (LP64)" begin + # Reverse OpenBLAS32_jll's LIBPATH_list so that we get the right openblas.so + run_all_tests("openblas", reverse(OpenBLAS32_jll.LIBPATH_list), :LP64, "") +end + +# Next, build a version that links against `libblastrampoline`, and tell +# the trampoline to forwards calls to `OpenBLAS_jll` +lbt_link_name, lbt_dir = build_libblastrampoline() +lbt_dir = joinpath(lbt_dir, binlib) + +@testset "LBT -> OpenBLAS_jll ($(openblas_interface))" begin + libdirs = unique(vcat(lbt_dir, OpenBLAS_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) + + # Test that setting bad `LBT_FORCE_*` values actually breaks things + # This can be somewhat unpredictable (segfaulting sometimes, returning zero other times) + # so it's hard to test on CI, so we comment it out for now. + #= + withenv("LBT_FORCE_RETSTYLE" => "ARGUMENT") do + cdotc_fail = ("cdotc_test", cdotc[2], false) + run_test(cdotc_fail, blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) + end + =# + + withenv("LBT_USE_RTLD_DEEPBIND" => "false") do + dgemm_deepbindless = ( + "dgemm_test", + ("||C||^2 is: 24.3384", "avoiding usage of RTLD_DEEPBIND"), + true, + ) + run_test(dgemm_deepbindless, blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) + end +end + +# And again, but this time with OpenBLAS32_jll +@testset "LBT -> OpenBLAS32_jll (LP64)" begin + libdirs = unique(vcat(lbt_dir, OpenBLAS32_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, :LP64, OpenBLAS32_jll.libopenblas_path) + + # Test that setting bad `LBT_FORCE_*` values actually breaks things + # but only on non-armv7l, because our `isamax` test is doubly broken there, due to a + # `int64_t` return being passed on the stack, thus being filled with half trash. + if Sys.ARCH != :arm + withenv("LBT_FORCE_INTERFACE" => "ILP64") do + # `max_idx: 2` is incorrect, it's what happens when ILP64 data is given to an LP64 backend + isamax_fail = ("isamax_test", ["max_idx: 2"], true) + run_test(isamax_fail, blastrampoline_link_name(), libdirs, :ILP64, OpenBLAS32_jll.libopenblas_path) + end + end +end + +# Finally the super-crazy test: build a binary that links against BOTH sets of symbols! +if openblas_interface == :ILP64 + inconsolable = ("inconsolable_test", ("||C||^2 is: 24.3384", "||b||^2 is: 3.0000"), true) + @testset "LBT -> OpenBLAS 32 + 64 (LP64 + ILP64)" begin + libdirs = unique(vcat(lbt_dir, OpenBLAS32_jll.LIBPATH_list..., OpenBLAS_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_test(inconsolable, lbt_link_name, libdirs, :wild_sobbing, "$(OpenBLAS32_jll.libopenblas_path);$(OpenBLAS_jll.libopenblas_path)") + end +end diff --git a/test/backends/refblas/Project.toml b/test/backends/refblas/Project.toml new file mode 100644 index 0000000..c7acb6d --- /dev/null +++ b/test/backends/refblas/Project.toml @@ -0,0 +1,7 @@ +[deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +LAPACK32_jll = "17f450c3-bd24-55df-bb84-8c51b4b939e3" +ReferenceBLAS32_jll = "9e84b91c-71b0-5f24-acdc-49dbe8049396" + +# ILP64 reference BLAS/LAPACK (ReferenceBLAS_jll / LAPACK_jll) are not yet usable through +# LBT's autodetection; see the commented-out ILP64 testset in `runtests.jl`. diff --git a/test/backends/refblas/runtests.jl b/test/backends/refblas/runtests.jl new file mode 100644 index 0000000..95c7825 --- /dev/null +++ b/test/backends/refblas/runtests.jl @@ -0,0 +1,35 @@ +include(joinpath(@__DIR__, "..", "..", "common.jl")) +using ReferenceBLAS32_jll, LAPACK32_jll, CompilerSupportLibraries_jll +# NB: `ReferenceBLAS_jll` / `LAPACK_jll` (ILP64) are intentionally not loaded here; see +# the commented-out ILP64 testset below. + +# Reference BLAS/LAPACK has no CBLAS layer, so we skip the `cdotc` test everywhere. + +lbt_link_name, lbt_dir = build_libblastrampoline() +lbt_dir = joinpath(lbt_dir, binlib) + +# --- LP64: ReferenceBLAS32_jll / LAPACK32_jll --- +@testset "Vanilla ReferenceBLAS32_jll (LP64)" begin + run_all_tests("blas32", reverse(ReferenceBLAS32_jll.LIBPATH_list), :LP64, "", tests = [dgemm, sdot]) +end + +@testset "LBT -> ReferenceBLAS32_jll / LAPACK32_jll (LP64)" begin + libdirs = unique(vcat(lbt_dir, ReferenceBLAS32_jll.LIBPATH_list..., LAPACK32_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) + run_all_tests(blastrampoline_link_name(), libdirs, :LP64, string(ReferenceBLAS32_jll.libblas32_path, ";", LAPACK32_jll.liblapack32_path); tests = [dgemm, dpstrf, sgesv, sdot]) +end + +# --- ILP64: ReferenceBLAS_jll / LAPACK_jll --- +# Disabled for now: the ILP64 `LAPACK_jll`/`ReferenceBLAS_jll` artifacts export *both* +# the plain (`ilaver_`) and the `_64_`-suffixed (`ilaver_64_`) symbols, so LBT's +# autodetection identifies them as LP64 and never forwards the ILP64 LAPACK symbols +# (the test then fails with `no BLAS/LAPACK library loaded for ilaver_()`). This needs +# the ILP64 reference libraries to be rebuilt in Yggdrasil with a proper `64_` symbol +# suffix (and `libblas64`/`liblapack64` sonames); tracked in +# https://github.com/JuliaLinearAlgebra/libblastrampoline/pull/128 +# +# if Sys.WORD_SIZE == 64 +# @testset "LBT -> ReferenceBLAS_jll / LAPACK_jll (ILP64)" begin +# libdirs = unique(vcat(lbt_dir, ReferenceBLAS_jll.LIBPATH_list..., LAPACK_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) +# run_all_tests(blastrampoline_link_name(), libdirs, :ILP64, string(ReferenceBLAS_jll.libblas_path, ";", LAPACK_jll.liblapack_path); tests = [dgemm, dpstrf, sgesv, sdot]) +# end +# end diff --git a/test/common.jl b/test/common.jl new file mode 100644 index 0000000..e9bc7c9 --- /dev/null +++ b/test/common.jl @@ -0,0 +1,112 @@ +# Shared test harness for all backends. Each backend's `runtests.jl` includes this +# file (which in turn includes `utils.jl`), then `using`s only the JLLs relevant to +# that backend. This keeps every backend in its own process with its own Project.toml +# so that only a single set of BLAS/LAPACK libraries is ever loaded at once. +using Pkg, Artifacts, Base.BinaryPlatforms, Libdl, Test + +include("utils.jl") + +# Compile `dgemm_test.c` and `sgesv_test.c` against the given BLAS/LAPACK +function run_test((test_name, test_expected_outputs, expect_success), libblas_name, libdirs, interface, backing_libs; extra_env = Dict()) + # We need to configure this C build a bit + cflags = String[ + "-g", + ] + if interface == :ILP64 + push!(cflags, "-DILP64") + end + + # If our GCC is 64-bit but Julia is 32-bit, pass -m32 + if needs_m32() + push!(cflags, "-m32") + end + + ldflags = String[ + # Teach it to find that libblas and its dependencies at build time + ("\"-L$(pathesc(libdir))\"" for libdir in libdirs)..., + "-l$(libblas_name)", + ] + + if !Sys.iswindows() + # Teach it to find that libblas and its dependencies at run time + append!(ldflags, ("-Wl,-rpath,$(pathesc(libdir))" for libdir in libdirs)) + end + + if Sys.islinux() + # Linux needs this for transitive dependencies + append!(ldflags, ("-Wl,-rpath-link,$(pathesc(libdir))" for libdir in libdirs)) + end + + mktempdir() do dir + @info("Compiling `$(test_name)` against $(libblas_name) ($(backing_libs)) in $(dir)") + srcdir = joinpath(@__DIR__, test_name) + make_cmd = `$(make) -sC $(pathesc(srcdir)) prefix=$(pathesc(dir)) CFLAGS="$(join(cflags, " "))" LDFLAGS="$(join(ldflags, " "))"` + p = run(ignorestatus(make_cmd)) + if !success(p) + @error("compilation failed", srcdir, prefix=dir, cflags=join(cflags, " "), ldflags=join(ldflags, " ")) + end + @test success(p) + + env = Dict( + # We need to tell it how to find CSL at run-time + LIBPATH_env => append_libpath(libdirs), + "LBT_DEFAULT_LIBS" => backing_libs, + "LBT_STRICT" => 1, + "LBT_VERBOSE" => 1, + pairs(extra_env)..., + ) + cmd = `$(dir)/$(test_name)` + p, output = capture_output(addenv(cmd, env)) + + expected_return_value = !xor(success(p), expect_success) + if !expected_return_value + @error("Test failed", env, p.exitcode, p.termsignal, expect_success) + println(output) + end + @test expected_return_value + + # Expect to see the path to `libblastrampoline` within the output, + # since we have `LBT_VERBOSE=1` and at startup, it announces its own path: + if startswith(libblas_name, "blastrampoline") && expect_success + lbt_libdir = first(libdirs) + @test occursin(lbt_libdir, output) + end + + # Test to make sure the test ran properly + has_expected_output = all(occursin(expected, output) for expected in test_expected_outputs) + if !has_expected_output + # Uh-oh, we didn't get what we expected. Time to debug! + @error("Test failed, got output:") + println(output) + + # If we're not on CI, launch `gdb` + if isempty(get(ENV, "CI", "")) + debugger = Sys.isbsd() ? "lldb" : "gdb" + @warn("Launching $debugger") + cmd = `$(debugger) $(cmd)` + run(addenv(cmd, env)) + end + end + @test has_expected_output + end +end + +# our tests, written in C, defined in subdirectories in `test/` +dgemm = ("dgemm_test", ("||C||^2 is: 24.3384",), true) +dgemmt = ("dgemmt_test", ("||C||^2 is: 23.2952",), true) +dpstrf = ("dpstrf_test", ("diag(A): 2.2601 1.8067 1.6970 0.4121",), true) +sgesv = ("sgesv_test", ("||b||^2 is: 3.0000",), true) +sgesv_failure = ("sgesv_test", ("Error: no BLAS/LAPACK library loaded!",), false) +sdot = ("sdot_test", ("C is: 1.9900",), true) +cdotc = ("cdotc_test", ( + "C (cblas) is: ( 1.4700, 3.8300)", + "C (fortran) is: ( 1.4700, 3.8300)", + ), true) + +# Helper function to run all the tests with the given arguments +# Does not include `dgemmt` because that's MKL-only +function run_all_tests(args...; tests = [dgemm, dpstrf, sgesv, sdot, cdotc], kwargs...) + for test in tests + run_test(test, args...; kwargs...) + end +end diff --git a/test/dpstrf_test/dpstrf_test.c b/test/dpstrf_test/dpstrf_test.c index e43a661..a151fc8 100644 --- a/test/dpstrf_test/dpstrf_test.c +++ b/test/dpstrf_test/dpstrf_test.c @@ -41,7 +41,11 @@ int main() // return value MANGLE(dpstrf_)("U", &order, &A[0][0], &lda, &pivot[0], &rank, &tol, &work[0], &info); if (info != 0) { - printf("ERROR: info == %ld!\n", info); + #ifdef ILP64 + printf("ERROR: info == %ld!\n", info); + #else + printf("ERROR: info == %d!\n", info); + #endif return 1; } diff --git a/test/runtests.jl b/test/runtests.jl index fa30492..c7861a1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,262 +1,54 @@ -using OpenBLAS_jll, OpenBLAS32_jll, MKL_jll, CompilerSupportLibraries_jll -using Pkg, Artifacts, Base.BinaryPlatforms, Libdl, Test - -include("utils.jl") - -# Compile `dgemm_test.c` and `sgesv_test.c` against the given BLAS/LAPACK -function run_test((test_name, test_expected_outputs, expect_success), libblas_name, libdirs, interface, backing_libs; extra_env = Dict()) - # We need to configure this C build a bit - cflags = String[ - "-g", - ] - if interface == :ILP64 - push!(cflags, "-DILP64") - end - - # If our GCC is 64-bit but Julia is 32-bit, pass -m32 - if needs_m32() - push!(cflags, "-m32") - end - - ldflags = String[ - # Teach it to find that libblas and its dependencies at build time - ("\"-L$(pathesc(libdir))\"" for libdir in libdirs)..., - "-l$(libblas_name)", - ] - - if !Sys.iswindows() - # Teach it to find that libblas and its dependencies at run time - append!(ldflags, ("-Wl,-rpath,$(pathesc(libdir))" for libdir in libdirs)) - end - - if Sys.islinux() - # Linux needs this for transitive dependencies - append!(ldflags, ("-Wl,-rpath-link,$(pathesc(libdir))" for libdir in libdirs)) - end - - mktempdir() do dir - @info("Compiling `$(test_name)` against $(libblas_name) ($(backing_libs)) in $(dir)") - srcdir = joinpath(@__DIR__, test_name) - make_cmd = `$(make) -sC $(pathesc(srcdir)) prefix=$(pathesc(dir)) CFLAGS="$(join(cflags, " "))" LDFLAGS="$(join(ldflags, " "))"` - p = run(ignorestatus(make_cmd)) - if !success(p) - @error("compilation failed", srcdir, prefix=dir, cflags=join(cflags, " "), ldflags=join(ldflags, " ")) - end - @test success(p) - - env = Dict( - # We need to tell it how to find CSL at run-time - LIBPATH_env => append_libpath(libdirs), - "LBT_DEFAULT_LIBS" => backing_libs, - "LBT_STRICT" => 1, - "LBT_VERBOSE" => 1, - pairs(extra_env)..., - ) - cmd = `$(dir)/$(test_name)` - p, output = capture_output(addenv(cmd, env)) - - expected_return_value = !xor(success(p), expect_success) - if !expected_return_value - @error("Test failed", env, p.exitcode, p.termsignal, expect_success) - println(output) - end - @test expected_return_value - - # Expect to see the path to `libblastrampoline` within the output, - # since we have `LBT_VERBOSE=1` and at startup, it announces its own path: - if startswith(libblas_name, "blastrampoline") && expect_success - lbt_libdir = first(libdirs) - @test occursin(lbt_libdir, output) - end - - # Test to make sure the test ran properly - has_expected_output = all(occursin(expected, output) for expected in test_expected_outputs) - if !has_expected_output - # Uh-oh, we didn't get what we expected. Time to debug! - @error("Test failed, got output:") - println(output) - - # If we're not on CI, launch `gdb` - if isempty(get(ENV, "CI", "")) - debugger = Sys.isbsd() ? "lldb" : "gdb" - @warn("Launching $debugger") - cmd = `$(debugger) $(cmd)` - run(addenv(cmd, env)) - end - end - @test has_expected_output - end -end - -# our tests, written in C, defined in subdirectories in `test/` -dgemm = ("dgemm_test", ("||C||^2 is: 24.3384",), true) -dgemmt = ("dgemmt_test", ("||C||^2 is: 23.2952",), true) -dpstrf = ("dpstrf_test", ("diag(A): 2.2601 1.8067 1.6970 0.4121",), true) -sgesv = ("sgesv_test", ("||b||^2 is: 3.0000",), true) -sgesv_failure = ("sgesv_test", ("Error: no BLAS/LAPACK library loaded!",), false) -sdot = ("sdot_test", ("C is: 1.9900",), true) -cdotc = ("cdotc_test", ( - "C (cblas) is: ( 1.4700, 3.8300)", - "C (fortran) is: ( 1.4700, 3.8300)", - ), true) - -# Helper function to run all the tests with the given arguments -# Does not include `dgemmt` because that's MKL-only -function run_all_tests(args...; tests = [dgemm, dpstrf, sgesv, sdot, cdotc], kwargs...) - for test in tests - run_test(test, args...; kwargs...) - end -end - -# Build version that links against vanilla OpenBLAS -openblas_interface = :LP64 -if Sys.WORD_SIZE == 64 - openblas_interface = :ILP64 -end -openblas_jll_libname = splitext(basename(OpenBLAS_jll.libopenblas_path)[4:end])[1] -@testset "Vanilla OpenBLAS_jll ($(openblas_interface))" begin - run_all_tests(openblas_jll_libname, OpenBLAS_jll.LIBPATH_list, openblas_interface, "") -end - -# Build version that links against vanilla OpenBLAS32 -@testset "Vanilla OpenBLAS32_jll (LP64)" begin - # Reverse OpenBLAS32_jll's LIBPATH_list so that we get the right openblas.so - run_all_tests("openblas", reverse(OpenBLAS32_jll.LIBPATH_list), :LP64, "") -end - -# Next, build a version that links against `libblastrampoline`, and tell -# the trampoline to forwards calls to `OpenBLAS_jll` -lbt_link_name, lbt_dir = build_libblastrampoline() -lbt_dir = joinpath(lbt_dir, binlib) - -@testset "LBT -> OpenBLAS_jll ($(openblas_interface))" begin - libdirs = unique(vcat(lbt_dir, OpenBLAS_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) - run_all_tests(blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) - - # Test that setting bad `LBT_FORCE_*` values actually breaks things - # This can be somewhat unpredictable (segfaulting sometimes, returning zero other times) - # so it's hard to test on CI, so we comment it out for now. - #= - withenv("LBT_FORCE_RETSTYLE" => "ARGUMENT") do - cdotc_fail = ("cdotc_test", cdotc[2], false) - run_test(cdotc_fail, blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) - end - =# - - withenv("LBT_USE_RTLD_DEEPBIND" => "false") do - dgemm_deepbindless = ( - "dgemm_test", - ("||C||^2 is: 24.3384", "avoiding usage of RTLD_DEEPBIND"), - true, - ) - run_test(dgemm_deepbindless, blastrampoline_link_name(), libdirs, openblas_interface, OpenBLAS_jll.libopenblas_path) +# Top-level test driver. Each BLAS/LAPACK backend lives in its own directory under +# `backends//` with its own `Project.toml` (pulling in only the JLLs relevant to +# that backend) and its own `runtests.jl`. +# +# Usage: +# julia --project=test/backends/openblas test/runtests.jl openblas +# Runs a single backend *in this process*, using the BLAS/LAPACK JLLs from the +# active project. This is how `.github/workflows/ci.yml` invokes each job. +# +# julia test/runtests.jl +# Runs every backend, each in its own process with its own project, so that only +# a single set of BLAS/LAPACK libraries is ever loaded at once. Handy locally. +# +# julia test/runtests.jl openblas mkl +# Runs the named backends, each in its own process/project (same isolation). +using Test + +const ALL_BACKENDS = ["openblas", "mkl", "refblas", "blis", "blas64", "accelerate", "direct"] + +selected_backends = isempty(ARGS) ? ALL_BACKENDS : ARGS +for backend in selected_backends + if !(backend in ALL_BACKENDS) + error("Unknown BLAS/LAPACK backend \"$(backend)\". Available backends: $(join(ALL_BACKENDS, ", "))") + end +end + +if length(selected_backends) == 1 && !isempty(ARGS) + # A single backend was explicitly requested: run it in *this* process. The caller is + # responsible for activating the matching `test/backends//Project.toml` so + # that only that backend's BLAS/LAPACK JLLs are loaded. + backend = only(selected_backends) + @info("Running test backend: $(backend)") + @testset "$(backend)" begin + include(joinpath(@__DIR__, "backends", backend, "runtests.jl")) + end +else + # Multiple (or all) backends: run each in its own process with its own project, so + # that only a single set of BLAS/LAPACK libraries is ever loaded at once. + julia = Base.julia_cmd() + failed = String[] + for backend in selected_backends + dir = joinpath(@__DIR__, "backends", backend) + @info("==================== Running backend: $(backend) ====================") + instantiate = `$(julia) --project=$(dir) -e "import Pkg; Pkg.instantiate()"` + testcmd = `$(julia) --project=$(dir) $(@__FILE__) $(backend)` + ok = success(run(ignorestatus(instantiate))) && success(run(ignorestatus(testcmd))) + if !ok + push!(failed, backend) + end + end + if !isempty(failed) + error("The following backends had test failures: $(join(failed, ", "))") end end - -# And again, but this time with OpenBLAS32_jll -@testset "LBT -> OpenBLAS32_jll (LP64)" begin - libdirs = unique(vcat(lbt_dir, OpenBLAS32_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) - run_all_tests(blastrampoline_link_name(), libdirs, :LP64, OpenBLAS32_jll.libopenblas_path) - - # Test that setting bad `LBT_FORCE_*` values actually breaks things - # but only on non-armv7l, because our `isamax` test is doubly broken there, due to a - # `int64_t` return being passed on the stack, thus being filled with half trash. - if Sys.ARCH != :arm - withenv("LBT_FORCE_INTERFACE" => "ILP64") do - # `max_idx: 2` is incorrect, it's what happens when ILP64 data is given to an LP64 backend - isamax_fail = ("isamax_test", ["max_idx: 2"], true) - run_test(isamax_fail, blastrampoline_link_name(), libdirs, :ILP64, OpenBLAS32_jll.libopenblas_path) - end - end -end - -# Test against MKL_jll using `libmkl_rt`, which is :LP64 by default -if MKL_jll.is_available() - # On i686, we can't do complex return style autodetection, so we manually set it, - # knowing that MKL is argument-style. - extra_env = Dict{String,String}() - if Sys.ARCH == :i686 - extra_env["LBT_FORCE_RETSTYLE"] = "ARGUMENT" - end - @testset "LBT -> MKL_jll (LP64)" begin - libdirs = unique(vcat(lbt_dir, MKL_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) - run_all_tests(blastrampoline_link_name(), libdirs, :LP64, MKL_jll.libmkl_rt_path; tests = [dgemm, dgemmt, dpstrf, sgesv, sdot, cdotc], extra_env) - end - - # Test that we can set MKL's interface via an environment variable to select ILP64, and LBT detects it properly - if Sys.WORD_SIZE == 64 - @testset "LBT -> MKL_jll (ILP64, via env)" begin - withenv("MKL_INTERFACE_LAYER" => "ILP64") do - libdirs = unique(vcat(lbt_dir, MKL_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) - run_all_tests(blastrampoline_link_name(), libdirs, :ILP64, MKL_jll.libmkl_rt_path; tests = [dgemm, dgemmt, dpstrf, sgesv, sdot, cdotc], extra_env) - end - end - end -end - -# Do we have Accelerate available? Note that we can't use `isfile()` here, since Apple -# has Helpfully (TM) sequestered the dynamic libraries away into a database somewhere that -# just gets fed into `dlopen()` when you ask for that magic path. -veclib_blas_path = "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/libBLAS.dylib" -if dlopen_e(veclib_blas_path) != C_NULL - # Test that we can run BLAS-only tests without LAPACK loaded (`sgesv` test requires LAPACK symbols) - @testset "LBT -> vecLib/libBLAS" begin - run_all_tests(blastrampoline_link_name(), [lbt_dir], :LP64, veclib_blas_path; tests=[dgemm, sdot, cdotc]) - end - - # With LAPACK as well, run all tests except `dgemmt` - veclib_lapack_path = "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/libLAPACK.dylib" - @testset "LBT -> vecLib/libLAPACK" begin - run_all_tests(blastrampoline_link_name(), [lbt_dir], :LP64, string(veclib_blas_path, ";", veclib_lapack_path)) - end - - veclib_lapack_handle = dlopen(veclib_lapack_path) - if dlsym_e(veclib_lapack_handle, "dpotrf\$NEWLAPACK\$ILP64") != C_NULL - @testset "LBT -> vecLib/libBLAS (ILP64)" begin - veclib_blas_path_ilp64 = "$(veclib_blas_path)!\x1a\$NEWLAPACK\$ILP64" - run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_blas_path_ilp64; tests=[dgemm, sdot, cdotc]) - end - - @testset "LBT -> vecLib/libLAPACK (ILP64)" begin - veclib_lapack_path_ilp64 = "$(veclib_lapack_path)!\x1a\$NEWLAPACK\$ILP64" - @warn("dpstrf test broken on new LAPACK in Accelerate") - dpstrf_broken = (dpstrf[1], "diag(A): 2.2601 1.7140 0.6206 1.1878", true) - run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, veclib_lapack_path_ilp64; tests=[dgemm, dpstrf_broken, sgesv, sdot, cdotc]) - end - end -end - - -# Do we have a `blas64.so` somewhere? If so, test with that for fun -blas64 = dlopen("libblas64", throw_error=false) -if blas64 !== nothing - # Test that we can run BLAS-only tests without LAPACK loaded (`sgesv` test requires LAPACK symbols, blas64 doesn't have CBLAS) - @testset "LBT -> libblas64 (ILP64, BLAS)" begin - run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, dlpath(blas64); tests=[dgemm, sdot]) - end - - # Check if we have a `liblapack` and if we do, run again, this time including `sgesv` - lapack = dlopen("liblapack64", throw_error=false) - if lapack !== nothing - @testset "LBT -> libblas64 + liblapack64 (ILP64, BLAS+LAPACK)" begin - run_all_tests(blastrampoline_link_name(), [lbt_dir], :ILP64, "$(dlpath(blas64));$(dlpath(lapack))"; tests=[dgemm, sdot, sgesv]) - end - end -end - -# Finally the super-crazy test: build a binary that links against BOTH sets of symbols! -if openblas_interface == :ILP64 - inconsolable = ("inconsolable_test", ("||C||^2 is: 24.3384", "||b||^2 is: 3.0000"), true) - @testset "LBT -> OpenBLAS 32 + 64 (LP64 + ILP64)" begin - libdirs = unique(vcat(lbt_dir, OpenBLAS32_jll.LIBPATH_list..., OpenBLAS_jll.LIBPATH_list..., CompilerSupportLibraries_jll.LIBPATH_list...)) - run_test(inconsolable, lbt_link_name, libdirs, :wild_sobbing, "$(OpenBLAS32_jll.libopenblas_path);$(OpenBLAS_jll.libopenblas_path)") - end -end - -# Run our "direct" tests within Julia -include("direct.jl") - -# Run some Apple Accelerate tests, but only on Apple -@static if Sys.isapple() - include("accelerate.jl") -end \ No newline at end of file diff --git a/test/utils.jl b/test/utils.jl index 8395eee..5a53c44 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -85,9 +85,13 @@ end # Build blastrampoline into a temporary directory, and return that blastrampoline_build_dir = nothing +# Fake linking name so that we can test from within Julia versions that actually load LBT natively. +const blastrampoline_dev_link_name = "blastramp-dev" function build_libblastrampoline() + # NB: always return the same `(link_name, build_dir)` tuple, whether we built just now + # or are returning the cached result, so callers can safely destructure it every time. if blastrampoline_build_dir !== nothing - return blastrampoline_build_dir + return blastrampoline_dev_link_name, blastrampoline_build_dir end cflags_add = "-Werror" * (needs_m32() ? " -m32" : "") @@ -97,8 +101,7 @@ function build_libblastrampoline() run(`$(make) -sC $(pathesc(srcdir)) CFLAGS="$(cflags_add)" ARCH=$(Sys.ARCH) clean`) run(`$(make) -sC $(pathesc(srcdir)) CFLAGS="$(cflags_add)" ARCH=$(Sys.ARCH) install builddir=$(pathesc(dir))/build prefix=$(pathesc(blastrampoline_build_dir))`) - # Give LBT a fake linking name so that we can test from within Julia versions that actually load LBT natively. - link_name = "blastramp-dev" + link_name = blastrampoline_dev_link_name cp( joinpath(blastrampoline_build_dir, binlib, blastrampoline_major_version()), joinpath(blastrampoline_build_dir, binlib, "lib$(link_name).$(shlib_ext)"), @@ -107,6 +110,14 @@ function build_libblastrampoline() return link_name, blastrampoline_build_dir end +# Build (if necessary) and `dlopen()` libblastrampoline for the in-process tests. +# We give it a fake linking name in `build_libblastrampoline()` so that we can test +# from within Julia versions that load LBT natively. +function open_lbt_handle() + lbt_link_name, lbt_prefix = build_libblastrampoline() + return dlopen("$(lbt_prefix)/$(binlib)/lib$(lbt_link_name).$(shlib_ext)", RTLD_GLOBAL | RTLD_DEEPBIND) +end + # Keep these in sync with `src/libblastrampoline_internal.h` struct lbt_library_info_t @@ -197,3 +208,30 @@ end function lbt_get_default_func(handle) return ccall(dlsym(handle, :lbt_get_default_func), Ptr{Cvoid}, ()) end + +# Helpers for inspecting an `lbt_config_t` from the in-process (`direct`/`accelerate`) tests +function unpack_loaded_libraries(config::lbt_config_t) + libs = LBTLibraryInfo[] + idx = 1 + lib_ptr = unsafe_load(config.loaded_libs, idx) + while lib_ptr != C_NULL + push!(libs, LBTLibraryInfo(unsafe_load(lib_ptr), config.num_exported_symbols)) + + idx += 1 + lib_ptr = unsafe_load(config.loaded_libs, idx) + end + return libs +end + +function find_symbol_offset(config::lbt_config_t, symbol::String) + for sym_idx in 1:config.num_exported_symbols + if unsafe_string(unsafe_load(config.exported_symbols, sym_idx)) == symbol + return UInt32(sym_idx - 1) + end + end + return nothing +end + +function bitfield_get(field::Vector{UInt8}, symbol_idx::UInt32) + return field[div(symbol_idx,8)+1] & (UInt8(0x01) << (symbol_idx%8)) +end