From 033035aad042bbf54705f63e39bf2c77b6a782a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 00:00:53 +0200 Subject: [PATCH 1/7] Add Dockerfiles --- docker/cuda/Dockerfile | 92 +++++++++++++++++++++++++++++++++ docker/cuda/docker-build.sh | 12 +++++ docker/cuda/fix_queue.patch | 45 ++++++++++++++++ docker/cuda/fix_sort.patch | 13 +++++ docker/cuda/fix_sparse.patch | 13 +++++ docker/opencl/Dockerfile | 93 ++++++++++++++++++++++++++++++++++ docker/opencl/docker-build.sh | 11 ++++ docker/opencl/fix_queue.patch | 45 ++++++++++++++++ docker/opencl/fix_sort.patch | 13 +++++ docker/opencl/fix_sparse.patch | 13 +++++ 10 files changed, 350 insertions(+) create mode 100644 docker/cuda/Dockerfile create mode 100755 docker/cuda/docker-build.sh create mode 100644 docker/cuda/fix_queue.patch create mode 100644 docker/cuda/fix_sort.patch create mode 100644 docker/cuda/fix_sparse.patch create mode 100644 docker/opencl/Dockerfile create mode 100755 docker/opencl/docker-build.sh create mode 100644 docker/opencl/fix_queue.patch create mode 100644 docker/opencl/fix_sort.patch create mode 100644 docker/opencl/fix_sparse.patch diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile new file mode 100644 index 0000000..0527a6a --- /dev/null +++ b/docker/cuda/Dockerfile @@ -0,0 +1,92 @@ +FROM nvidia/cuda:10.1-devel-ubuntu18.04 AS khiva-cuda-builder-base +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + cmake \ + git \ + libboost-all-dev \ + libfftw3-dev \ + libfreeimage-dev \ + liblapack-dev \ + liblapacke-dev \ + libopenblas-dev \ + openjdk-8-jdk \ + python3-pip && \ + pip3 install setuptools && \ + pip3 install conan --upgrade && \ + conan remote add conan-mpusz https://api.bintray.com/conan/mpusz/conan-mpusz && \ + conan profile new --detect default && \ + conan profile update settings.compiler.libcxx=libstdc++11 default && \ + rm -rf /var/lib/apt/lists/* + +# AF_DISABLE_GRAPHICS - Environment variable to disable graphics at +# runtime due to lack of graphics support by docker - visit +# http://arrayfire.org/docs/configuring_environment.htm#af_disable_graphics +# for more information +# For build instructions: https://github.com/arrayfire/arrayfire/wiki/Build-Instructions-for-Linux +ENV CXX g++-7 +ENV CC gcc-7 +WORKDIR /root + +COPY *.patch ./ +FROM khiva-cuda-builder-base AS arrayfire-cuda-builder +ENV AF_DISABLE_GRAPHICS=1 +RUN git clone --recursive https://github.com/arrayfire/arrayfire.git && \ + cd arrayfire && \ + git checkout tags/v3.6.2 && \ + git apply /root/fix_queue.patch && \ + git apply /root/fix_sort.patch && \ + git apply /root/fix_sparse.patch && \ + mkdir build && cd build && \ + cmake .. -DCMAKE_INSTALL_PREFIX=/opt/arrayfire \ + -DCMAKE_BUILD_TYPE=Release \ + -DAF_BUILD_CPU=ON \ + -DAF_BUILD_CUDA=ON \ + -DAF_BUILD_OPENCL=OFF \ + -DCUDA_cublas_device_LIBRARY=/usr/local/cuda/lib64 \ + -DAF_BUILD_UNIFIED=ON \ + -DAF_WITH_GRAPHICS=OFF \ + -DAF_WITH_NONFREE=OFF \ + -DAF_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=ON \ + -DAF_BUILD_DOCS=OFF \ + -DAF_WITH_LOGGING=OFF \ + -DAF_WITH_STATIC_FREEIMAGE=OFF && \ + cmake --build . --target install -- -j8 + +FROM khiva-cuda-builder-base AS khiva-cuda-builder +COPY --from=arrayfire-cuda-builder /opt/arrayfire /opt/arrayfire +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ + ldconfig && \ + git clone --recurse-submodules https://github.com/shapelets/khiva.git && \ + cd khiva && \ + git checkout tags/v0.5.0 && \ + mkdir build && \ + cd build && \ + conan install .. --build missing && \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/khiva \ + -DKHIVA_BUILD_TESTS=ON \ + -DKHIVA_BUILD_BENCHMARKS=ON \ + -DKHIVA_BUILD_EXAMPLES=OFF \ + -DKHIVA_BUILD_DOCUMENTATION=OFF \ + -DKHIVA_USE_CONAN=ON \ + -DKHIVA_BUILD_JNI_BINDINGS=ON && \ + cmake --build . --target install -- -j8 + +FROM nvidia/cuda:10.1-runtime-ubuntu18.04 AS arrayfire-cuda +COPY --from=arrayfire-cuda-builder /opt/arrayfire /opt/arrayfire +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libopenblas-base \ + libfftw3-3 \ + liblapacke && \ + echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ + ldconfig + +FROM arrayfire-cuda AS khiva-cuda +COPY --from=khiva-cuda-builder /opt/khiva /opt/khiva +RUN echo /opt/khiva/lib > /etc/ld.so.conf.d/khiva.conf && \ + ldconfig diff --git a/docker/cuda/docker-build.sh b/docker/cuda/docker-build.sh new file mode 100755 index 0000000..2b9ab5c --- /dev/null +++ b/docker/cuda/docker-build.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +docker build --target khiva-cuda-builder-base -t $PUSH_REGISTRY/shapelets/khiva-cuda-builder-base:0.5.0 . +docker build --target arrayfire-cuda-builder -t $PUSH_REGISTRY/shapelets/arrayfire-cuda-builder:3.6.2 . +docker build --target khiva-cuda-builder -t $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 . +docker run --gpus all --workdir /root/khiva/build $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 ctest --output-on-failure -j8 +docker build --target arrayfire-cuda -t $PUSH_REGISTRY/shapelets/arrayfire-cuda:3.6.2 . +docker build --target khiva-cuda -t $PUSH_REGISTRY/shapelets/khiva-cuda:0.5.0 . +docker push $PUSH_REGISTRY/shapelets/khiva-cuda-builder-base:0.5.0 +docker push $PUSH_REGISTRY/shapelets/arrayfire-cuda-builder:3.6.2 +docker push $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 +docker push $PUSH_REGISTRY/shapelets/arrayfire-cuda:3.6.2 +docker push $PUSH_REGISTRY/shapelets/khiva-cuda:0.5.0 diff --git a/docker/cuda/fix_queue.patch b/docker/cuda/fix_queue.patch new file mode 100644 index 0000000..6215240 --- /dev/null +++ b/docker/cuda/fix_queue.patch @@ -0,0 +1,45 @@ +diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp +index 3c502400..a5aab7d6 100644 +--- a/src/backend/cpu/queue.hpp ++++ b/src/backend/cpu/queue.hpp +@@ -11,39 +11,9 @@ + #include + #include + +-//FIXME: Is there a better way to check for std::future not being supported ? +-#if defined(AF_DISABLE_CPU_ASYNC) || (defined(__GNUC__) && (__GCC_ATOMIC_INT_LOCK_FREE < 2 || __GCC_ATOMIC_POINTER_LOCK_FREE < 2)) +- +-#include +-using std::function; +-#include +-#define __SYNCHRONOUS_ARCH 1 +-class queue_impl +-{ +-public: +- template +- void enqueue(const F func, Args... args) const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- } +- +- void sync() const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- } +- +- bool is_worker() const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- return false; +- } +- +-}; +- +-#else +- + #include + #define __SYNCHRONOUS_ARCH 0 +-typedef async_queue queue_impl; +- +-#endif ++typedef threads::async_queue queue_impl; + + #pragma once + diff --git a/docker/cuda/fix_sort.patch b/docker/cuda/fix_sort.patch new file mode 100644 index 0000000..9b4afc9 --- /dev/null +++ b/docker/cuda/fix_sort.patch @@ -0,0 +1,13 @@ +diff --git a/src/backend/cpu/kernel/sort.hpp b/src/backend/cpu/kernel/sort.hpp +index dd842dce..4f2a8853 100644 +--- a/src/backend/cpu/kernel/sort.hpp ++++ b/src/backend/cpu/kernel/sort.hpp +@@ -27,7 +27,7 @@ void sort0Iterative(Param val, bool isAscending) + // initialize original index locations + T *val_ptr = val.get(); + +- function op = std::greater(); ++ std::function op = std::greater(); + if(isAscending) { op = std::less(); } + + T *comp_ptr = nullptr; diff --git a/docker/cuda/fix_sparse.patch b/docker/cuda/fix_sparse.patch new file mode 100644 index 0000000..48298c2 --- /dev/null +++ b/docker/cuda/fix_sparse.patch @@ -0,0 +1,13 @@ +diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp +index d6032ade..302bd798 100644 +--- a/src/backend/cpu/sparse.cpp ++++ b/src/backend/cpu/sparse.cpp +@@ -339,7 +339,7 @@ SparseArray sparseConvertStorageToStorage(const SparseArray &in) + SparseArray converted = createEmptySparseArray(in.dims(), (int)in.getNNZ(), dest); + converted.eval(); + +- function, Param, Param, ++ std::function, Param, Param, + CParam, CParam, CParam) + > converter; + diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile new file mode 100644 index 0000000..3256dd2 --- /dev/null +++ b/docker/opencl/Dockerfile @@ -0,0 +1,93 @@ +FROM intelopencl/intel-opencl:ubuntu-18.04-ppa AS khiva-opencl-builder-base +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + libboost-all-dev \ + libfftw3-dev \ + libfreeimage-dev \ + liblapack-dev \ + liblapacke-dev \ + libopenblas-dev \ + ocl-icd-opencl-dev \ + python3-pip \ + software-properties-common && \ + pip3 install setuptools && \ + pip3 install conan --upgrade && \ + conan remote add conan-mpusz https://api.bintray.com/conan/mpusz/conan-mpusz && \ + conan profile new --detect default && \ + conan profile update settings.compiler.libcxx=libstdc++11 default && \ + rm -rf /var/lib/apt/lists/* + +ENV CXX g++-7 +ENV CC gcc-7 +WORKDIR /root +COPY *.patch ./ + +# AF_DISABLE_GRAPHICS - Environment variable to disable graphics at +# runtime due to lack of graphics support by docker - visit +# http://arrayfire.org/docs/configuring_environment.htm#af_disable_graphics +# for more information +# For build instructions: https://github.com/arrayfire/arrayfire/wiki/Build-Instructions-for-Linux +FROM khiva-opencl-builder-base AS arrayfire-opencl-builder +ENV AF_DISABLE_GRAPHICS=1 +RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/arrayfire/arrayfire.git && \ + cd arrayfire && \ + git apply /root/fix_queue.patch && \ + git apply /root/fix_sort.patch && \ + git apply /root/fix_sparse.patch && \ + mkdir build && cd build && \ + cmake .. -DCMAKE_INSTALL_PREFIX=/opt/arrayfire \ + -DCMAKE_BUILD_TYPE=Release \ + -DAF_BUILD_CPU=ON \ + -DAF_BUILD_CUDA=OFF \ + -DAF_BUILD_OPENCL=ON \ + -DAF_BUILD_UNIFIED=ON \ + -DAF_WITH_GRAPHICS=OFF \ + -DAF_WITH_NONFREE=OFF \ + -DAF_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=ON \ + -DAF_BUILD_DOCS=OFF \ + -DAF_WITH_LOGGING=OFF \ + -DAF_WITH_STATIC_FREEIMAGE=OFF && \ + cmake --build . --target install -- -j8 + +FROM khiva-opencl-builder-base AS khiva-builder +ARG KHIVA_BRANCH=v0.5.0 +WORKDIR /root +COPY --from=arrayfire-opencl-builder /opt/arrayfire /opt/arrayfire +RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ + ldconfig && \ + add-apt-repository ppa:intel-opencl/intel-opencl && \ + apt-get update && \ + apt-get install -y intel-opencl-icd clinfo && \ + git clone -b v0.5.0 --depth 1 --recurse-submodules -j 8 https://github.com/shapelets/khiva.git && \ + cd khiva && \ + mkdir build && \ + cd build && \ + conan install .. --build missing && \ + cmake .. -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/khiva \ + -DKHIVA_BUILD_TESTS=ON \ + -DKHIVA_BUILD_BENCHMARKS=ON \ + -DKHIVA_BUILD_EXAMPLES=OFF \ + -DKHIVA_BUILD_DOCUMENTATION=OFF \ + -DKHIVA_USE_CONAN=ON \ + -DKHIVA_BUILD_JNI_BINDINGS=OFF && \ + cmake --build . --target install -- -j8 + +FROM intelopencl/intel-opencl:ubuntu-18.04-ppa AS arrayfire-intel-opencl +COPY --from=arrayfire-opencl-builder /opt/arrayfire /opt/arrayfire +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libopenblas-base \ + libfftw3-3 \ + liblapacke && \ + echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ + ldconfig + +FROM arrayfire-intel-opencl as khiva-intel-opencl +COPY --from=khiva-builder /opt/khiva /opt/khiva +RUN echo /opt/khiva/lib > /etc/ld.so.conf.d/khiva.conf && \ + ldconfig \ No newline at end of file diff --git a/docker/opencl/docker-build.sh b/docker/opencl/docker-build.sh new file mode 100755 index 0000000..c277a71 --- /dev/null +++ b/docker/opencl/docker-build.sh @@ -0,0 +1,11 @@ +#! /usr/bin/env bash +docker build --target khiva-opencl-builder-base -t $PUSH_REGISTRY/shapelets/khiva-opencl-builder-base:0.5.0 . +docker build --target arrayfire-opencl-builder -t $PUSH_REGISTRY/shapelets/arrayfire-opencl-builder:3.6.2 . +docker build --target khiva-builder -t $PUSH_REGISTRY/shapelets/khiva-builder:0.5.0 . +docker build --target arrayfire-intel-opencl -t $PUSH_REGISTRY/shapelets/arrayfire-intel-opencl:3.6.2 . +docker build --target khiva-intel-opencl -t $PUSH_REGISTRY/shapelets/khiva:0.5.0-opencl . +docker push $PUSH_REGISTRY/shapelets/khiva-opencl-builder-base:0.5.0 +docker push $PUSH_REGISTRY/shapelets/arrayfire-opencl-builder:3.6.2 +docker push $PUSH_REGISTRY/shapelets/khiva-builder:0.5.0 +docker push $PUSH_REGISTRY/shapelets/arrayfire-intel-opencl:3.6.2 +docker push $PUSH_REGISTRY/shapelets/khiva:0.5.0-opencl \ No newline at end of file diff --git a/docker/opencl/fix_queue.patch b/docker/opencl/fix_queue.patch new file mode 100644 index 0000000..6215240 --- /dev/null +++ b/docker/opencl/fix_queue.patch @@ -0,0 +1,45 @@ +diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp +index 3c502400..a5aab7d6 100644 +--- a/src/backend/cpu/queue.hpp ++++ b/src/backend/cpu/queue.hpp +@@ -11,39 +11,9 @@ + #include + #include + +-//FIXME: Is there a better way to check for std::future not being supported ? +-#if defined(AF_DISABLE_CPU_ASYNC) || (defined(__GNUC__) && (__GCC_ATOMIC_INT_LOCK_FREE < 2 || __GCC_ATOMIC_POINTER_LOCK_FREE < 2)) +- +-#include +-using std::function; +-#include +-#define __SYNCHRONOUS_ARCH 1 +-class queue_impl +-{ +-public: +- template +- void enqueue(const F func, Args... args) const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- } +- +- void sync() const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- } +- +- bool is_worker() const { +- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); +- return false; +- } +- +-}; +- +-#else +- + #include + #define __SYNCHRONOUS_ARCH 0 +-typedef async_queue queue_impl; +- +-#endif ++typedef threads::async_queue queue_impl; + + #pragma once + diff --git a/docker/opencl/fix_sort.patch b/docker/opencl/fix_sort.patch new file mode 100644 index 0000000..9b4afc9 --- /dev/null +++ b/docker/opencl/fix_sort.patch @@ -0,0 +1,13 @@ +diff --git a/src/backend/cpu/kernel/sort.hpp b/src/backend/cpu/kernel/sort.hpp +index dd842dce..4f2a8853 100644 +--- a/src/backend/cpu/kernel/sort.hpp ++++ b/src/backend/cpu/kernel/sort.hpp +@@ -27,7 +27,7 @@ void sort0Iterative(Param val, bool isAscending) + // initialize original index locations + T *val_ptr = val.get(); + +- function op = std::greater(); ++ std::function op = std::greater(); + if(isAscending) { op = std::less(); } + + T *comp_ptr = nullptr; diff --git a/docker/opencl/fix_sparse.patch b/docker/opencl/fix_sparse.patch new file mode 100644 index 0000000..48298c2 --- /dev/null +++ b/docker/opencl/fix_sparse.patch @@ -0,0 +1,13 @@ +diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp +index d6032ade..302bd798 100644 +--- a/src/backend/cpu/sparse.cpp ++++ b/src/backend/cpu/sparse.cpp +@@ -339,7 +339,7 @@ SparseArray sparseConvertStorageToStorage(const SparseArray &in) + SparseArray converted = createEmptySparseArray(in.dims(), (int)in.getNNZ(), dest); + converted.eval(); + +- function, Param, Param, ++ std::function, Param, Param, + CParam, CParam, CParam) + > converter; + From 2e400bd8fc970801a787a2166e407acd85c45b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 00:13:02 +0200 Subject: [PATCH 2/7] Add KHIVA_BRANCH build-arg --- docker/cuda/Dockerfile | 11 +++++------ docker/cuda/docker-build.sh | 2 +- docker/opencl/Dockerfile | 8 ++++---- docker/opencl/docker-build.sh | 4 ++-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile index 0527a6a..b9ae342 100644 --- a/docker/cuda/Dockerfile +++ b/docker/cuda/Dockerfile @@ -31,10 +31,9 @@ WORKDIR /root COPY *.patch ./ FROM khiva-cuda-builder-base AS arrayfire-cuda-builder ENV AF_DISABLE_GRAPHICS=1 -RUN git clone --recursive https://github.com/arrayfire/arrayfire.git && \ +RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/arrayfire/arrayfire.git && \ cd arrayfire && \ - git checkout tags/v3.6.2 && \ - git apply /root/fix_queue.patch && \ + git apply /root/fix_queue.patch && \ git apply /root/fix_sort.patch && \ git apply /root/fix_sparse.patch && \ mkdir build && cd build && \ @@ -55,14 +54,14 @@ RUN git clone --recursive https://github.com/arrayfire/arrayfire.git && \ cmake --build . --target install -- -j8 FROM khiva-cuda-builder-base AS khiva-cuda-builder +ARG KHIVA_BRANCH=v0.5.0 COPY --from=arrayfire-cuda-builder /opt/arrayfire /opt/arrayfire ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ ldconfig && \ - git clone --recurse-submodules https://github.com/shapelets/khiva.git && \ + git clone -b ${KHIVA_BRANCH} --depth 1 --recurse-submodules -j 8 https://github.com/shapelets/khiva.git && \ cd khiva && \ - git checkout tags/v0.5.0 && \ - mkdir build && \ + mkdir build && \ cd build && \ conan install .. --build missing && \ cmake .. \ diff --git a/docker/cuda/docker-build.sh b/docker/cuda/docker-build.sh index 2b9ab5c..73eb326 100755 --- a/docker/cuda/docker-build.sh +++ b/docker/cuda/docker-build.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash docker build --target khiva-cuda-builder-base -t $PUSH_REGISTRY/shapelets/khiva-cuda-builder-base:0.5.0 . docker build --target arrayfire-cuda-builder -t $PUSH_REGISTRY/shapelets/arrayfire-cuda-builder:3.6.2 . -docker build --target khiva-cuda-builder -t $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 . +docker build --target khiva-cuda-builder --build-arg KHIVA_BRANCH=v0.5.0 -t $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 . docker run --gpus all --workdir /root/khiva/build $PUSH_REGISTRY/shapelets/khiva-cuda-builder:0.5.0 ctest --output-on-failure -j8 docker build --target arrayfire-cuda -t $PUSH_REGISTRY/shapelets/arrayfire-cuda:3.6.2 . docker build --target khiva-cuda -t $PUSH_REGISTRY/shapelets/khiva-cuda:0.5.0 . diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile index 3256dd2..891f3c0 100644 --- a/docker/opencl/Dockerfile +++ b/docker/opencl/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update && \ liblapacke-dev \ libopenblas-dev \ ocl-icd-opencl-dev \ - python3-pip \ + python3-pip \ software-properties-common && \ pip3 install setuptools && \ pip3 install conan --upgrade && \ @@ -51,7 +51,7 @@ RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/a -DAF_BUILD_DOCS=OFF \ -DAF_WITH_LOGGING=OFF \ -DAF_WITH_STATIC_FREEIMAGE=OFF && \ - cmake --build . --target install -- -j8 + cmake --build . --target install -- -j8 FROM khiva-opencl-builder-base AS khiva-builder ARG KHIVA_BRANCH=v0.5.0 @@ -62,7 +62,7 @@ RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ add-apt-repository ppa:intel-opencl/intel-opencl && \ apt-get update && \ apt-get install -y intel-opencl-icd clinfo && \ - git clone -b v0.5.0 --depth 1 --recurse-submodules -j 8 https://github.com/shapelets/khiva.git && \ + git clone -b ${KHIVA_BRANCH} --depth 1 --recurse-submodules -j 8 https://github.com/shapelets/khiva.git && \ cd khiva && \ mkdir build && \ cd build && \ @@ -90,4 +90,4 @@ RUN apt-get update && \ FROM arrayfire-intel-opencl as khiva-intel-opencl COPY --from=khiva-builder /opt/khiva /opt/khiva RUN echo /opt/khiva/lib > /etc/ld.so.conf.d/khiva.conf && \ - ldconfig \ No newline at end of file + ldconfig diff --git a/docker/opencl/docker-build.sh b/docker/opencl/docker-build.sh index c277a71..9a73145 100755 --- a/docker/opencl/docker-build.sh +++ b/docker/opencl/docker-build.sh @@ -1,11 +1,11 @@ #! /usr/bin/env bash docker build --target khiva-opencl-builder-base -t $PUSH_REGISTRY/shapelets/khiva-opencl-builder-base:0.5.0 . docker build --target arrayfire-opencl-builder -t $PUSH_REGISTRY/shapelets/arrayfire-opencl-builder:3.6.2 . -docker build --target khiva-builder -t $PUSH_REGISTRY/shapelets/khiva-builder:0.5.0 . +docker build --target khiva-builder --build-arg KHIVA_BRANCH=v0.5.0 -t $PUSH_REGISTRY/shapelets/khiva-builder:0.5.0 . docker build --target arrayfire-intel-opencl -t $PUSH_REGISTRY/shapelets/arrayfire-intel-opencl:3.6.2 . docker build --target khiva-intel-opencl -t $PUSH_REGISTRY/shapelets/khiva:0.5.0-opencl . docker push $PUSH_REGISTRY/shapelets/khiva-opencl-builder-base:0.5.0 docker push $PUSH_REGISTRY/shapelets/arrayfire-opencl-builder:3.6.2 docker push $PUSH_REGISTRY/shapelets/khiva-builder:0.5.0 docker push $PUSH_REGISTRY/shapelets/arrayfire-intel-opencl:3.6.2 -docker push $PUSH_REGISTRY/shapelets/khiva:0.5.0-opencl \ No newline at end of file +docker push $PUSH_REGISTRY/shapelets/khiva:0.5.0-opencl From 44d585704b869a39b13da9b0f0ef646f414c1885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 13:58:26 +0200 Subject: [PATCH 3/7] Update Dockerfile --- docker/cuda/Dockerfile | 1 + docker/opencl/Dockerfile | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile index b9ae342..410bba5 100644 --- a/docker/cuda/Dockerfile +++ b/docker/cuda/Dockerfile @@ -72,6 +72,7 @@ RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ -DKHIVA_BUILD_EXAMPLES=OFF \ -DKHIVA_BUILD_DOCUMENTATION=OFF \ -DKHIVA_USE_CONAN=ON \ + -DKHIVA_BUILD_C_BINDINGS=ON \ -DKHIVA_BUILD_JNI_BINDINGS=ON && \ cmake --build . --target install -- -j8 diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile index 891f3c0..dabf336 100644 --- a/docker/opencl/Dockerfile +++ b/docker/opencl/Dockerfile @@ -4,12 +4,13 @@ RUN apt-get update && \ build-essential \ cmake \ git \ - libboost-all-dev \ + libboost-math-dev \ libfftw3-dev \ libfreeimage-dev \ liblapack-dev \ liblapacke-dev \ libopenblas-dev \ + openjdk-8-jdk \ ocl-icd-opencl-dev \ python3-pip \ software-properties-common && \ @@ -74,7 +75,8 @@ RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ -DKHIVA_BUILD_EXAMPLES=OFF \ -DKHIVA_BUILD_DOCUMENTATION=OFF \ -DKHIVA_USE_CONAN=ON \ - -DKHIVA_BUILD_JNI_BINDINGS=OFF && \ + -DKHIVA_BUILD_C_BINDINGS=ON \ + -DKHIVA_BUILD_JNI_BINDINGS=ON && \ cmake --build . --target install -- -j8 FROM intelopencl/intel-opencl:ubuntu-18.04-ppa AS arrayfire-intel-opencl From 5cc755fbb1c89c7294b9166446f6bb703574a447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 16:09:59 +0200 Subject: [PATCH 4/7] Improve build time of containers --- docker/cuda/Dockerfile | 7 ++----- docker/opencl/Dockerfile | 5 +---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile index 410bba5..3d97706 100644 --- a/docker/cuda/Dockerfile +++ b/docker/cuda/Dockerfile @@ -31,11 +31,8 @@ WORKDIR /root COPY *.patch ./ FROM khiva-cuda-builder-base AS arrayfire-cuda-builder ENV AF_DISABLE_GRAPHICS=1 -RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/arrayfire/arrayfire.git && \ +RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/arrayfire/arrayfire.git && \ cd arrayfire && \ - git apply /root/fix_queue.patch && \ - git apply /root/fix_sort.patch && \ - git apply /root/fix_sparse.patch && \ mkdir build && cd build && \ cmake .. -DCMAKE_INSTALL_PREFIX=/opt/arrayfire \ -DCMAKE_BUILD_TYPE=Release \ @@ -47,7 +44,7 @@ RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/ -DAF_WITH_GRAPHICS=OFF \ -DAF_WITH_NONFREE=OFF \ -DAF_BUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=ON \ + -DBUILD_TESTING=OFF \ -DAF_BUILD_DOCS=OFF \ -DAF_WITH_LOGGING=OFF \ -DAF_WITH_STATIC_FREEIMAGE=OFF && \ diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile index dabf336..01f0cc2 100644 --- a/docker/opencl/Dockerfile +++ b/docker/opencl/Dockerfile @@ -35,9 +35,6 @@ FROM khiva-opencl-builder-base AS arrayfire-opencl-builder ENV AF_DISABLE_GRAPHICS=1 RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/arrayfire/arrayfire.git && \ cd arrayfire && \ - git apply /root/fix_queue.patch && \ - git apply /root/fix_sort.patch && \ - git apply /root/fix_sparse.patch && \ mkdir build && cd build && \ cmake .. -DCMAKE_INSTALL_PREFIX=/opt/arrayfire \ -DCMAKE_BUILD_TYPE=Release \ @@ -48,7 +45,7 @@ RUN git clone -b v3.6.2 --depth 1 --recurse-submodules -j 8 https://github.com/a -DAF_WITH_GRAPHICS=OFF \ -DAF_WITH_NONFREE=OFF \ -DAF_BUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=ON \ + -DBUILD_TESTING=OFF \ -DAF_BUILD_DOCS=OFF \ -DAF_WITH_LOGGING=OFF \ -DAF_WITH_STATIC_FREEIMAGE=OFF && \ From 5f89ef5451cfe26e8ac725c144baa6d21f14473f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 17:01:36 +0200 Subject: [PATCH 5/7] Improve space of containers --- docker/cuda/Dockerfile | 6 +++--- docker/opencl/Dockerfile | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile index 3d97706..b178efb 100644 --- a/docker/cuda/Dockerfile +++ b/docker/cuda/Dockerfile @@ -14,10 +14,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-pip && \ pip3 install setuptools && \ pip3 install conan --upgrade && \ - conan remote add conan-mpusz https://api.bintray.com/conan/mpusz/conan-mpusz && \ conan profile new --detect default && \ conan profile update settings.compiler.libcxx=libstdc++11 default && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* # AF_DISABLE_GRAPHICS - Environment variable to disable graphics at # runtime due to lack of graphics support by docker - visit @@ -81,7 +80,8 @@ RUN apt-get update && \ libfftw3-3 \ liblapacke && \ echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ - ldconfig + ldconfig && \ + rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* FROM arrayfire-cuda AS khiva-cuda COPY --from=khiva-cuda-builder /opt/khiva /opt/khiva diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile index 01f0cc2..9f60763 100644 --- a/docker/opencl/Dockerfile +++ b/docker/opencl/Dockerfile @@ -16,10 +16,9 @@ RUN apt-get update && \ software-properties-common && \ pip3 install setuptools && \ pip3 install conan --upgrade && \ - conan remote add conan-mpusz https://api.bintray.com/conan/mpusz/conan-mpusz && \ conan profile new --detect default && \ conan profile update settings.compiler.libcxx=libstdc++11 default && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* ENV CXX g++-7 ENV CC gcc-7 @@ -57,9 +56,6 @@ WORKDIR /root COPY --from=arrayfire-opencl-builder /opt/arrayfire /opt/arrayfire RUN echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ ldconfig && \ - add-apt-repository ppa:intel-opencl/intel-opencl && \ - apt-get update && \ - apt-get install -y intel-opencl-icd clinfo && \ git clone -b ${KHIVA_BRANCH} --depth 1 --recurse-submodules -j 8 https://github.com/shapelets/khiva.git && \ cd khiva && \ mkdir build && \ @@ -84,7 +80,8 @@ RUN apt-get update && \ libfftw3-3 \ liblapacke && \ echo /opt/arrayfire/lib > /etc/ld.so.conf.d/arrayfire.conf && \ - ldconfig + ldconfig && \ + rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* FROM arrayfire-intel-opencl as khiva-intel-opencl COPY --from=khiva-builder /opt/khiva /opt/khiva From a2a0bfbd8e54db42612c57f25cd5ee1b70e783af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Tue, 5 May 2020 19:49:47 +0200 Subject: [PATCH 6/7] Remove unused patch files --- docker/cuda/fix_queue.patch | 45 ---------------------------------- docker/cuda/fix_sort.patch | 13 ---------- docker/cuda/fix_sparse.patch | 13 ---------- docker/opencl/fix_queue.patch | 45 ---------------------------------- docker/opencl/fix_sort.patch | 13 ---------- docker/opencl/fix_sparse.patch | 13 ---------- 6 files changed, 142 deletions(-) delete mode 100644 docker/cuda/fix_queue.patch delete mode 100644 docker/cuda/fix_sort.patch delete mode 100644 docker/cuda/fix_sparse.patch delete mode 100644 docker/opencl/fix_queue.patch delete mode 100644 docker/opencl/fix_sort.patch delete mode 100644 docker/opencl/fix_sparse.patch diff --git a/docker/cuda/fix_queue.patch b/docker/cuda/fix_queue.patch deleted file mode 100644 index 6215240..0000000 --- a/docker/cuda/fix_queue.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp -index 3c502400..a5aab7d6 100644 ---- a/src/backend/cpu/queue.hpp -+++ b/src/backend/cpu/queue.hpp -@@ -11,39 +11,9 @@ - #include - #include - --//FIXME: Is there a better way to check for std::future not being supported ? --#if defined(AF_DISABLE_CPU_ASYNC) || (defined(__GNUC__) && (__GCC_ATOMIC_INT_LOCK_FREE < 2 || __GCC_ATOMIC_POINTER_LOCK_FREE < 2)) -- --#include --using std::function; --#include --#define __SYNCHRONOUS_ARCH 1 --class queue_impl --{ --public: -- template -- void enqueue(const F func, Args... args) const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- } -- -- void sync() const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- } -- -- bool is_worker() const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- return false; -- } -- --}; -- --#else -- - #include - #define __SYNCHRONOUS_ARCH 0 --typedef async_queue queue_impl; -- --#endif -+typedef threads::async_queue queue_impl; - - #pragma once - diff --git a/docker/cuda/fix_sort.patch b/docker/cuda/fix_sort.patch deleted file mode 100644 index 9b4afc9..0000000 --- a/docker/cuda/fix_sort.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/backend/cpu/kernel/sort.hpp b/src/backend/cpu/kernel/sort.hpp -index dd842dce..4f2a8853 100644 ---- a/src/backend/cpu/kernel/sort.hpp -+++ b/src/backend/cpu/kernel/sort.hpp -@@ -27,7 +27,7 @@ void sort0Iterative(Param val, bool isAscending) - // initialize original index locations - T *val_ptr = val.get(); - -- function op = std::greater(); -+ std::function op = std::greater(); - if(isAscending) { op = std::less(); } - - T *comp_ptr = nullptr; diff --git a/docker/cuda/fix_sparse.patch b/docker/cuda/fix_sparse.patch deleted file mode 100644 index 48298c2..0000000 --- a/docker/cuda/fix_sparse.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp -index d6032ade..302bd798 100644 ---- a/src/backend/cpu/sparse.cpp -+++ b/src/backend/cpu/sparse.cpp -@@ -339,7 +339,7 @@ SparseArray sparseConvertStorageToStorage(const SparseArray &in) - SparseArray converted = createEmptySparseArray(in.dims(), (int)in.getNNZ(), dest); - converted.eval(); - -- function, Param, Param, -+ std::function, Param, Param, - CParam, CParam, CParam) - > converter; - diff --git a/docker/opencl/fix_queue.patch b/docker/opencl/fix_queue.patch deleted file mode 100644 index 6215240..0000000 --- a/docker/opencl/fix_queue.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp -index 3c502400..a5aab7d6 100644 ---- a/src/backend/cpu/queue.hpp -+++ b/src/backend/cpu/queue.hpp -@@ -11,39 +11,9 @@ - #include - #include - --//FIXME: Is there a better way to check for std::future not being supported ? --#if defined(AF_DISABLE_CPU_ASYNC) || (defined(__GNUC__) && (__GCC_ATOMIC_INT_LOCK_FREE < 2 || __GCC_ATOMIC_POINTER_LOCK_FREE < 2)) -- --#include --using std::function; --#include --#define __SYNCHRONOUS_ARCH 1 --class queue_impl --{ --public: -- template -- void enqueue(const F func, Args... args) const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- } -- -- void sync() const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- } -- -- bool is_worker() const { -- AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL); -- return false; -- } -- --}; -- --#else -- - #include - #define __SYNCHRONOUS_ARCH 0 --typedef async_queue queue_impl; -- --#endif -+typedef threads::async_queue queue_impl; - - #pragma once - diff --git a/docker/opencl/fix_sort.patch b/docker/opencl/fix_sort.patch deleted file mode 100644 index 9b4afc9..0000000 --- a/docker/opencl/fix_sort.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/backend/cpu/kernel/sort.hpp b/src/backend/cpu/kernel/sort.hpp -index dd842dce..4f2a8853 100644 ---- a/src/backend/cpu/kernel/sort.hpp -+++ b/src/backend/cpu/kernel/sort.hpp -@@ -27,7 +27,7 @@ void sort0Iterative(Param val, bool isAscending) - // initialize original index locations - T *val_ptr = val.get(); - -- function op = std::greater(); -+ std::function op = std::greater(); - if(isAscending) { op = std::less(); } - - T *comp_ptr = nullptr; diff --git a/docker/opencl/fix_sparse.patch b/docker/opencl/fix_sparse.patch deleted file mode 100644 index 48298c2..0000000 --- a/docker/opencl/fix_sparse.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp -index d6032ade..302bd798 100644 ---- a/src/backend/cpu/sparse.cpp -+++ b/src/backend/cpu/sparse.cpp -@@ -339,7 +339,7 @@ SparseArray sparseConvertStorageToStorage(const SparseArray &in) - SparseArray converted = createEmptySparseArray(in.dims(), (int)in.getNNZ(), dest); - converted.eval(); - -- function, Param, Param, -+ std::function, Param, Param, - CParam, CParam, CParam) - > converter; - From 6ad0a76409d90bf1860dbec544e9328725a817ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Bocanegra?= Date: Wed, 27 May 2020 00:29:22 +0200 Subject: [PATCH 7/7] Update dockerfiles --- docker/cuda/Dockerfile | 1 + docker/opencl/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile index b178efb..4660529 100644 --- a/docker/cuda/Dockerfile +++ b/docker/cuda/Dockerfile @@ -84,6 +84,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* FROM arrayfire-cuda AS khiva-cuda +ENV LD_LIBRARY_PATH="/opt/khiva/lib:${LD_LIBRARY_PATH}" COPY --from=khiva-cuda-builder /opt/khiva /opt/khiva RUN echo /opt/khiva/lib > /etc/ld.so.conf.d/khiva.conf && \ ldconfig diff --git a/docker/opencl/Dockerfile b/docker/opencl/Dockerfile index 9f60763..f47a685 100644 --- a/docker/opencl/Dockerfile +++ b/docker/opencl/Dockerfile @@ -84,6 +84,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* FROM arrayfire-intel-opencl as khiva-intel-opencl +ENV LD_LIBRARY_PATH="/opt/khiva/lib:${LD_LIBRARY_PATH}" COPY --from=khiva-builder /opt/khiva /opt/khiva RUN echo /opt/khiva/lib > /etc/ld.so.conf.d/khiva.conf && \ ldconfig