diff --git a/.cirrus.yml b/.cirrus.yml index db1b71e2f..fc68ca1b0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,8 +16,8 @@ task: script: - brew install libomp - - brew install python@3.12 - - $(brew --prefix python@3.12)/bin/python3.12 -m venv .test_venv + - brew install python@3.11 + - $(brew --prefix python@3.11)/bin/python3.11 -m venv .test_venv - source .test_venv/bin/activate - pip install --upgrade pip - sh ./pytest.sh diff --git a/keopscore/keopscore/config/base_config.py b/keopscore/keopscore/config/base_config.py index fdabf91ba..3519bc5d2 100644 --- a/keopscore/keopscore/config/base_config.py +++ b/keopscore/keopscore/config/base_config.py @@ -5,7 +5,7 @@ import shutil from pathlib import Path import keopscore -from keopscore.utils.misc_utils import KeOps_Warning, KeOps_OS_Run +from keopscore.utils.misc_utils import KeOps_Warning, KeOps_OS_Run, get_brew_prefix from keopscore.utils.misc_utils import CHECK_MARK, CROSS_MARK @@ -219,48 +219,13 @@ def print_compile_options(self): """Print the compile options.""" print(f"Compile Options: {self.compile_options}") - def get_brew_prefix(self): - """Get Homebrew prefix path using KeOps_OS_Run""" - if platform.system() != "Darwin": - return None - - # Redirect brew --prefix to a temporary file - tmp_file = "/tmp/brew_prefix.txt" - - # brew --prefix > /tmp/brew_prefix.txt - # We use shell redirection so the output ends up in the file - KeOps_OS_Run(f"brew --prefix > {tmp_file}") - - # Now read the file if it was created - if os.path.exists(tmp_file): - with open(tmp_file, "r") as f: - prefix = f.read().strip() - - # Optional: Clean up - os.remove(tmp_file) - - # Return the prefix if it's non-empty - return prefix if prefix else None - - # If file doesn't exist or is empty, return None - return None - def get_use_Apple_clang(self): """Detect if using Apple Clang.""" is_apple_clang = False if platform.system() == "Darwin": - tmp_file = "/tmp/compiler_version.txt" - # Run "c++ --version" and write output to /tmp/compiler_version.txt - KeOps_OS_Run(f"c++ --version > {tmp_file}") - - # Now read the file - if os.path.exists(tmp_file): - with open(tmp_file, "r") as f: - compiler_info = f.read() - os.remove(tmp_file) - - # Check if 'Apple clang' appears in the output - is_apple_clang = "Apple clang" in compiler_info + compiler_info = KeOps_OS_Run(f"c++ --version").stdout.decode("utf-8") + # Check if 'Apple clang' appears in the output + is_apple_clang = "Apple clang" in compiler_info return is_apple_clang def set_cpp_flags(self): @@ -272,10 +237,11 @@ def set_cpp_flags(self): # Add OpenMP flags based on compiler if self.get_use_Apple_clang(): # For Apple Clang, you need to specify OpenMP library location - brew_prefix = self.get_brew_prefix() - self.cpp_flags += f" -Xpreprocessor -fopenmp" - self.cpp_flags += f" -I{brew_prefix}/opt/libomp/include" - self.cpp_flags += f" -L{brew_prefix}/opt/libomp/lib" + brew_prefix = get_brew_prefix() + if brew_prefix is not None: + self.cpp_flags += f" -Xpreprocessor -fopenmp" + self.cpp_flags += f" -I{brew_prefix}/opt/libomp/include" + self.cpp_flags += f" -L{brew_prefix}/opt/libomp/lib" else: # For GCC and other compilers self.cpp_flags += " -fopenmp" diff --git a/keopscore/keopscore/config/cuda.py b/keopscore/keopscore/config/cuda.py index a52e2b5b1..320f700e9 100644 --- a/keopscore/keopscore/config/cuda.py +++ b/keopscore/keopscore/config/cuda.py @@ -16,13 +16,12 @@ import shutil from os.path import join import platform -import tempfile import subprocess import sys import keopscore from keopscore.utils.misc_utils import KeOps_Warning from keopscore.utils.misc_utils import KeOps_OS_Run -from keopscore.utils.misc_utils import CHECK_MARK, CROSS_MARK +from keopscore.utils.misc_utils import CHECK_MARK, CROSS_MARK, get_include_file_abspath class CUDAConfig: @@ -341,17 +340,7 @@ def get_cuda_include_path(self): return self.cuda_include_path def get_include_file_abspath(self, filename): - tmp_file = tempfile.NamedTemporaryFile(dir=self.get_build_folder()).name - KeOps_OS_Run( - f'echo "#include <{filename}>" | {self.cxx_compiler} -M -E -x c++ - | head -n 2 > {tmp_file}' - ) - strings = open(tmp_file).read().split() - abspath = None - for s in strings: - if filename in s: - abspath = s - os.remove(tmp_file) - return abspath + return get_include_file_abspath(filename, self.cxx_compiler) def set_nvrtc_flags(self): """Set the NVRTC flags for CUDA compilation.""" diff --git a/keopscore/keopscore/config/openmp.py b/keopscore/keopscore/config/openmp.py index 2e3cc61c9..66ab72317 100644 --- a/keopscore/keopscore/config/openmp.py +++ b/keopscore/keopscore/config/openmp.py @@ -5,8 +5,7 @@ import platform from ctypes.util import find_library -from keopscore.utils.misc_utils import KeOps_Warning -from keopscore.utils.misc_utils import KeOps_OS_Run +from keopscore.utils.misc_utils import KeOps_Warning, KeOps_OS_Run, get_brew_prefix from keopscore.utils.misc_utils import CHECK_MARK, CROSS_MARK @@ -84,32 +83,6 @@ def check_compiler_for_openmp(self): os.remove(test_file) return False - def get_brew_prefix(self): - """Get Homebrew prefix path using KeOps_OS_Run""" - if platform.system() != "Darwin": - return None - - # Redirect brew --prefix to a temporary file - tmp_file = "/tmp/brew_prefix.txt" - - # brew --prefix > /tmp/brew_prefix.txt - # We use shell redirection so the output ends up in the file - KeOps_OS_Run(f"brew --prefix > {tmp_file}") - - # Now read the file if it was created - if os.path.exists(tmp_file): - with open(tmp_file, "r") as f: - prefix = f.read().strip() - - # Optional: Clean up - os.remove(tmp_file) - - # Return the prefix if it's non-empty - return prefix if prefix else None - - # If file doesn't exist or is empty, return None - return None - def check_openmp_libraries(self): if self.os.startswith("Linux"): openmp_lib = find_library("gomp") @@ -121,10 +94,13 @@ def check_openmp_libraries(self): return True # Specific check for M1/M2/M3 apple Silicon chips elif self.os.startswith("Darwin") and platform.machine() in ["arm64", "arm64e"]: - brew_prefix = self.get_brew_prefix() - openmp_path = f"{brew_prefix}/opt/libomp/lib/libomp.dylib" - openmp_lib = openmp_path if os.path.exists(openmp_path) else None - if not openmp_lib: + brew_prefix = get_brew_prefix() + if brew_prefix is not None: + openmp_path = f"{brew_prefix}/opt/libomp/lib/libomp.dylib" + openmp_lib = openmp_path if os.path.exists(openmp_path) else None + else: + openmp_lib = None + if openmp_lib is None: KeOps_Warning( "OpenMP library not found, it must be downloaded through Homebrew for apple Silicon chips" ) diff --git a/keopscore/keopscore/utils/gpu_utils.py b/keopscore/keopscore/utils/gpu_utils.py index fef910b87..df5b1debc 100644 --- a/keopscore/keopscore/utils/gpu_utils.py +++ b/keopscore/keopscore/utils/gpu_utils.py @@ -1,6 +1,5 @@ import ctypes from ctypes.util import find_library -import tempfile from keopscore.utils.misc_utils import ( @@ -8,6 +7,7 @@ KeOps_Warning, find_library_abspath, KeOps_OS_Run, + get_include_file_abspath, ) import keopscore @@ -81,7 +81,7 @@ def get_cuda_include_path(): # last try, testing if by any chance the header is already in the default # include path of gcc - path_cudah = get_include_file_abspath("cuda.h") + path_cudah = get_include_file_abspath("cuda.h", config.cxx_compiler()) if path_cudah: path = os.path.dirname(path_cudah) if os.path.isfile(join(path, "nvrtc.h")): @@ -100,20 +100,6 @@ def get_cuda_include_path(): ) -def get_include_file_abspath(filename): - tmp_file = tempfile.NamedTemporaryFile(dir=config.get_build_folder()).name - KeOps_OS_Run( - f'echo "#include <{filename}>" | {config.cxx_compiler()} -M -E -x c++ - | head -n 2 > {tmp_file}' - ) - strings = open(tmp_file).read().split() - abspath = None - for s in strings: - if filename in s: - abspath = s - os.remove(tmp_file) - return abspath - - def orig_cuda_include_fp16_path(): """ We look for float 16 cuda headers cuda_fp16.h and cuda_fp16.hpp diff --git a/keopscore/keopscore/utils/misc_utils.py b/keopscore/keopscore/utils/misc_utils.py index f48b102c4..887127dba 100644 --- a/keopscore/keopscore/utils/misc_utils.py +++ b/keopscore/keopscore/utils/misc_utils.py @@ -5,6 +5,7 @@ import keopscore from os.path import join import re +import platform def KeOps_Print(message, force_print=False, **kwargs): @@ -35,7 +36,7 @@ def KeOps_Error(message, show_line_number=True): raise ValueError(message) -def KeOps_OS_Run(command): +def KeOps_OS_Run(command, print_warning=True): import sys python_version = sys.version_info @@ -43,20 +44,41 @@ def KeOps_OS_Run(command): import subprocess out = subprocess.run(command, shell=True, capture_output=True) - if out.stderr != b"": + if out.stderr != b"" and print_warning: KeOps_Warning("There were warnings or errors :", newline=True) KeOps_Print(out.stderr.decode("utf-8")) elif python_version >= (3, 5): import subprocess - subprocess.run( + out = subprocess.run( command, shell=True, ) else: - import os + KeOps_Error("Python version >= 3.5 required.", newline=True) + return out + - os.system(command) +def get_include_file_abspath(filename, compiler): + out = KeOps_OS_Run( + f'echo "#include <{filename}>" | {compiler} -M -E -x c++ - | head -n 2' + ) + strings = out.stdout.decode("utf8").split() + abspath = None + for s in strings: + if filename in s: + abspath = s + return abspath + + +def get_brew_prefix(): + """Get Homebrew prefix path using KeOps_OS_Run""" + if platform.system() != "Darwin": + return None + out = KeOps_OS_Run(f"brew --prefix", print_warning=False) + if out.stderr != b"": + return None + return out.stdout.decode("utf-8").strip() def find_library_abspath(lib): diff --git a/pytest.sh b/pytest.sh index 0f746f8be..3b56ed1e3 100755 --- a/pytest.sh +++ b/pytest.sh @@ -64,7 +64,7 @@ PROJDIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) PYTHON="python3" # python environment for test -TEST_VENV=${PROJDIR}/.test_venv +TEST_VENV=${PROJDIR}/.test_venv_pytest # python test requirements (names of packages to be installed with pip) TEST_REQ="pip"