diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..d713fa8c61 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Shell scripts must use LF so they run on Linux CI runners. +*.sh text eol=lf diff --git a/.github/scripts/cleanup_pr_body.sh b/.github/scripts/cleanup_pr_body.sh index 409025bce6..25af344aab 100755 --- a/.github/scripts/cleanup_pr_body.sh +++ b/.github/scripts/cleanup_pr_body.sh @@ -1,50 +1,50 @@ -#!/bin/bash - -set -eu - -# ensure 1 argument is passed -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -PR_NUMBER=$1 -OLD=/tmp/orig_pr_body.txt -NEW=/tmp/new_pr_body.txt - -gh pr view --json body --template "{{.body}}" "${PR_NUMBER}" > "${OLD}" -cp "${OLD}" "${NEW}" - -# Remove markdown comments (like the at the start) -sed -i '/$/d' "${NEW}" - -# Remove "PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED." -sed -i '/PLEASE FILL IN THE PR DESCRIPTION HERE.*$/d' "${NEW}" - -# Remove all lines after and including "**BEFORE SUBMITTING, PLEASE READ THE CHECKLIST BELOW AND FILL IN THE DESCRIPTION ABOVE**" -sed -i '/\*\*BEFORE SUBMITTING, PLEASE READ.*\*\*/,$d' "${NEW}" - -# Remove HTML
section that includes text of "PR Checklist (Click to Expand)" -python3 - <.*?.*?PR Checklist \(Click to Expand\).*?.*?
', re.DOTALL) -content = re.sub(pattern, '', content) - -with open("${NEW}", "w") as file: - file.write(content) -EOF - -# Run this only if ${NEW} is different than ${OLD} -if ! cmp -s "${OLD}" "${NEW}"; then - gh pr edit --body-file "${NEW}" "${PR_NUMBER}" - echo - echo "Updated PR body:" - echo - cat "${NEW}" -else - echo "No changes needed" -fi +#!/bin/bash + +set -eu + +# ensure 1 argument is passed +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +PR_NUMBER=$1 +OLD=/tmp/orig_pr_body.txt +NEW=/tmp/new_pr_body.txt + +gh pr view --json body --template "{{.body}}" "${PR_NUMBER}" > "${OLD}" +cp "${OLD}" "${NEW}" + +# Remove markdown comments (like the at the start) +sed -i '/$/d' "${NEW}" + +# Remove "PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED." +sed -i '/PLEASE FILL IN THE PR DESCRIPTION HERE.*$/d' "${NEW}" + +# Remove all lines after and including "**BEFORE SUBMITTING, PLEASE READ THE CHECKLIST BELOW AND FILL IN THE DESCRIPTION ABOVE**" +sed -i '/\*\*BEFORE SUBMITTING, PLEASE READ.*\*\*/,$d' "${NEW}" + +# Remove HTML
section that includes text of "PR Checklist (Click to Expand)" +python3 - <.*?.*?PR Checklist \(Click to Expand\).*?.*?
', re.DOTALL) +content = re.sub(pattern, '', content) + +with open("${NEW}", "w") as file: + file.write(content) +EOF + +# Run this only if ${NEW} is different than ${OLD} +if ! cmp -s "${OLD}" "${NEW}"; then + gh pr edit --body-file "${NEW}" "${PR_NUMBER}" + echo + echo "Updated PR body:" + echo + cat "${NEW}" +else + echo "No changes needed" +fi diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build.sh index 06f535098d..c69ebbb42d 100644 --- a/.github/workflows/scripts/build.sh +++ b/.github/workflows/scripts/build.sh @@ -1,22 +1,22 @@ -#!/bin/bash -set -eux - -python_executable=python$1 -cuda_home=/usr/local/cuda-$2 - -# Update paths -PATH=${cuda_home}/bin:$PATH -LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH - -# Install requirements -$python_executable -m pip install -r requirements/build.txt -r requirements/cuda.txt - -# Limit the number of parallel jobs to avoid OOM -export MAX_JOBS=1 -# Make sure release wheels are built for the following architectures -export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX" - -bash tools/check_repo.sh - -# Build -$python_executable setup.py bdist_wheel --dist-dir=dist +#!/bin/bash +set -eux + +python_executable=python$1 +cuda_home=/usr/local/cuda-$2 + +# Update paths +PATH=${cuda_home}/bin:$PATH +LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH + +# Install requirements +$python_executable -m pip install -r requirements/build.txt -r requirements/cuda.txt + +# Limit the number of parallel jobs to avoid OOM +export MAX_JOBS=1 +# Make sure release wheels are built for the following architectures +export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX" + +bash tools/check_repo.sh + +# Build +$python_executable setup.py bdist_wheel --dist-dir=dist diff --git a/.github/workflows/scripts/cuda-install.sh b/.github/workflows/scripts/cuda-install.sh index 737942beff..3d0b7a1fe0 100644 --- a/.github/workflows/scripts/cuda-install.sh +++ b/.github/workflows/scripts/cuda-install.sh @@ -1,23 +1,23 @@ -#!/bin/bash - -# Replace '.' with '-' ex: 11.8 -> 11-8 -cuda_version=$(echo "$1" | tr "." "-") -# Removes '-' and '.' ex: ubuntu-20.04 -> ubuntu2004 -OS=$(echo "$2" | tr -d ".\-") - -# Installs CUDA -wget -nv "https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb" -sudo dpkg -i cuda-keyring_1.1-1_all.deb -rm cuda-keyring_1.1-1_all.deb -sudo apt -qq update -sudo apt -y install "cuda-${cuda_version}" "cuda-nvcc-${cuda_version}" "cuda-libraries-dev-${cuda_version}" -sudo apt clean - -# Test nvcc -PATH=/usr/local/cuda-$1/bin:${PATH} -nvcc --version - -# Log gcc, g++, c++ versions -gcc --version -g++ --version -c++ --version +#!/bin/bash + +# Replace '.' with '-' ex: 11.8 -> 11-8 +cuda_version=$(echo "$1" | tr "." "-") +# Removes '-' and '.' ex: ubuntu-20.04 -> ubuntu2004 +OS=$(echo "$2" | tr -d ".\-") + +# Installs CUDA +wget -nv "https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb" +sudo dpkg -i cuda-keyring_1.1-1_all.deb +rm cuda-keyring_1.1-1_all.deb +sudo apt -qq update +sudo apt -y install "cuda-${cuda_version}" "cuda-nvcc-${cuda_version}" "cuda-libraries-dev-${cuda_version}" +sudo apt clean + +# Test nvcc +PATH=/usr/local/cuda-$1/bin:${PATH} +nvcc --version + +# Log gcc, g++, c++ versions +gcc --version +g++ --version +c++ --version diff --git a/.github/workflows/scripts/env.sh b/.github/workflows/scripts/env.sh index ce815a0732..d7baaecbbc 100644 --- a/.github/workflows/scripts/env.sh +++ b/.github/workflows/scripts/env.sh @@ -1,56 +1,56 @@ -#!/bin/bash - -# This file installs common linux environment tools - -export LANG C.UTF-8 - -# python_version=$1 - -sudo apt-get update && \ -sudo apt-get install -y --no-install-recommends \ - software-properties-common \ - -sudo apt-get install -y --no-install-recommends \ - build-essential \ - apt-utils \ - ca-certificates \ - wget \ - git \ - vim \ - libssl-dev \ - curl \ - unzip \ - unrar \ - cmake \ - net-tools \ - sudo \ - autotools-dev \ - rsync \ - jq \ - openssh-server \ - tmux \ - screen \ - htop \ - pdsh \ - openssh-client \ - lshw \ - dmidecode \ - util-linux \ - automake \ - autoconf \ - libtool \ - net-tools \ - pciutils \ - libpci-dev \ - libaio-dev \ - libcap2 \ - libtinfo5 \ - fakeroot \ - devscripts \ - debhelper \ - nfs-common - -# Remove github bloat files to free up disk space -sudo rm -rf "/usr/local/share/boost" -sudo rm -rf "$AGENT_TOOLSDIRECTORY" -sudo rm -rf "/usr/share/dotnet" +#!/bin/bash + +# This file installs common linux environment tools + +export LANG C.UTF-8 + +# python_version=$1 + +sudo apt-get update && \ +sudo apt-get install -y --no-install-recommends \ + software-properties-common \ + +sudo apt-get install -y --no-install-recommends \ + build-essential \ + apt-utils \ + ca-certificates \ + wget \ + git \ + vim \ + libssl-dev \ + curl \ + unzip \ + unrar \ + cmake \ + net-tools \ + sudo \ + autotools-dev \ + rsync \ + jq \ + openssh-server \ + tmux \ + screen \ + htop \ + pdsh \ + openssh-client \ + lshw \ + dmidecode \ + util-linux \ + automake \ + autoconf \ + libtool \ + net-tools \ + pciutils \ + libpci-dev \ + libaio-dev \ + libcap2 \ + libtinfo5 \ + fakeroot \ + devscripts \ + debhelper \ + nfs-common + +# Remove github bloat files to free up disk space +sudo rm -rf "/usr/local/share/boost" +sudo rm -rf "$AGENT_TOOLSDIRECTORY" +sudo rm -rf "/usr/share/dotnet" diff --git a/.github/workflows/scripts/pytorch-install.sh b/.github/workflows/scripts/pytorch-install.sh index 30b2139eab..e3cda7dad2 100644 --- a/.github/workflows/scripts/pytorch-install.sh +++ b/.github/workflows/scripts/pytorch-install.sh @@ -1,15 +1,15 @@ -#!/bin/bash - -python_executable=python$1 -pytorch_version=$2 -cuda_version=$3 - -# Install torch -$python_executable -m pip install numpy pyyaml scipy ipython mkl mkl-include ninja cython typing pandas typing-extensions dataclasses setuptools && conda clean -ya -$python_executable -m pip install torch=="${pytorch_version}+cu${cuda_version//./}" --extra-index-url "https://download.pytorch.org/whl/cu${cuda_version//./}" - -# Print version information -$python_executable --version -$python_executable -c "import torch; print('PyTorch:', torch.__version__)" -$python_executable -c "import torch; print('CUDA:', torch.version.cuda)" -$python_executable -c "from torch.utils import cpp_extension; print (cpp_extension.CUDA_HOME)" +#!/bin/bash + +python_executable=python$1 +pytorch_version=$2 +cuda_version=$3 + +# Install torch +$python_executable -m pip install numpy pyyaml scipy ipython mkl mkl-include ninja cython typing pandas typing-extensions dataclasses setuptools && conda clean -ya +$python_executable -m pip install torch=="${pytorch_version}+cu${cuda_version//./}" --extra-index-url "https://download.pytorch.org/whl/cu${cuda_version//./}" + +# Print version information +$python_executable --version +$python_executable -c "import torch; print('PyTorch:', torch.__version__)" +$python_executable -c "import torch; print('CUDA:', torch.version.cuda)" +$python_executable -c "from torch.utils import cpp_extension; print (cpp_extension.CUDA_HOME)"