diff --git a/jupyterhub/Dockerfile b/jupyterhub/Dockerfile index 70d3de3..7e49e7e 100644 --- a/jupyterhub/Dockerfile +++ b/jupyterhub/Dockerfile @@ -1,5 +1,8 @@ # Custom image, see README -FROM jupyter/custom +# usually we build the fresh image of jupyter custom from docker stacks repo but since we are just updating a package +# here we will build it on top of our existing jupyter image +# FROM jupyter/custom +FROM us-docker.pkg.dev/vgi-pn-277619/data-team/pubnative/jupyterhub:spark3.4.1-py3.11-hadoop3-openjdk11-scala2.12-build2.0.0 USER root WORKDIR /usr/local/ diff --git a/jupyterhub/Makefile b/jupyterhub/Makefile index 46b4f43..4207f95 100644 --- a/jupyterhub/Makefile +++ b/jupyterhub/Makefile @@ -23,4 +23,4 @@ build: requirements publish: build docker push ${JUPYTERHUB} - @echo pushed ${JUPYTERHUB} to DockerHub + @echo pushed ${JUPYTERHUB} to Google Artifact Registry diff --git a/jupyterhub/README.md b/jupyterhub/README.md index 5b5b1a5..dc63921 100644 --- a/jupyterhub/README.md +++ b/jupyterhub/README.md @@ -1,17 +1,42 @@ -## Build base image +## Choosing a python version +- see available versions +```shell +conda search python +``` +- create conda environment. We are choosing python 3.11 mainly because tensorflow currently does not support python larger than 3.11 +![img.png](img.png) + +## Setting up environment +```shell +conda create -n jupyterhub python=3.11.7 conda +``` +```shell +conda activate jupyterhub +``` +```shell +poetry install +``` + +## Build base image first (jupyter/custom) for building jupyterhub image This image is built on top of this [repo](https://github.com/jupyter/docker-stacks). To reproduce follow the steps below: - Clone repo -- Build the image(`jupyter/custom`) using the command below +```shell +git clone git@github.com:jupyter/docker-stacks.git ``` +- Build the image(`jupyter/custom`) using the command below + +```shell docker build --rm --force-rm \ -t jupyter/custom ./images/pyspark-notebook \ +--build-arg python_version=3.11.10 \ --build-arg openjdk_version=11 \ --build-arg spark_version=3.4.1 \ --build-arg hadoop_version=3 \ --build-arg spark_download_url="https://archive.apache.org/dist/spark/" \ --platform linux/amd64 ``` +- If you are just adding a package instead of starting from `jupyter/custom` consider starting from your existing/current image of jupyter to minimize changes to user experience ## Additional information - `gcs-connector-hadoop3-2.2.11-shaded.jar` is added to spark diff --git a/jupyterhub/img.png b/jupyterhub/img.png new file mode 100644 index 0000000..bb5e939 Binary files /dev/null and b/jupyterhub/img.png differ diff --git a/jupyterhub/poetry.lock b/jupyterhub/poetry.lock index 0e0fec8..1f10fd5 100644 --- a/jupyterhub/poetry.lock +++ b/jupyterhub/poetry.lock @@ -433,27 +433,6 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] -[[package]] -name = "clarabel" -version = "0.6.0" -description = "Clarabel Conic Interior Point Solver for Rust / Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "clarabel-0.6.0-cp37-abi3-macosx_10_7_x86_64.whl", hash = "sha256:4f366de79b8bc66bef8dc170987840b672ccab9222e710c09536d78ef47f606d"}, - {file = "clarabel-0.6.0-cp37-abi3-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:edcebbfc14073cd32bfb664317fd2555716c96be8b2a54efdb2b728453582bea"}, - {file = "clarabel-0.6.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e737d2818b9ca10e92ccd3fa9ad1a805b039976016415a0c45adef3427d70792"}, - {file = "clarabel-0.6.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e0b1891d8e507eb0bfc7e0b981584c388b2ab28658056e600997dbbc23f1ab4"}, - {file = "clarabel-0.6.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9946d3b5db346421b6d839d868e7b1151b590f871344fe95113bfd55b5be2433"}, - {file = "clarabel-0.6.0-cp37-abi3-win32.whl", hash = "sha256:73ed408c975a8ea021c3d8262d5d023a18e1ac3f6bb59a37cd69a11dba8f86ed"}, - {file = "clarabel-0.6.0-cp37-abi3-win_amd64.whl", hash = "sha256:5a6be4df9fed98b6f73f034836def913a1ecd52e8b79ca230ddf7cd66ebcdee7"}, - {file = "clarabel-0.6.0.tar.gz", hash = "sha256:ef909a393e72981ca10b1d866d9cc7fb6295ece20ae035def764338894961184"}, -] - -[package.dependencies] -numpy = "*" -scipy = "*" - [[package]] name = "cloudpickle" version = "3.0.0" @@ -556,64 +535,6 @@ mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.6.1)", "types-Pill test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] -[[package]] -name = "cvxpy" -version = "1.4.1" -description = "A domain-specific language for modeling convex optimization problems in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cvxpy-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:03588055b660c043848f5281fe24dbd21f005b34bd8bd3b56906d8ad457c14ae"}, - {file = "cvxpy-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:315609ff96adeda4970471b349bc19d44ff4043e15630cf5ac70c029658fe8fc"}, - {file = "cvxpy-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55e08ffb973d62b3fabc675ad464cb6013ea5ce69799f330b33a084a2e580d8d"}, - {file = "cvxpy-1.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f1482558b785f2db51c76b9c6e91cc85dbd146675b126a799e7d7aab5b15354"}, - {file = "cvxpy-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:2f84687d15d11f9b49ca902f20103a2076efd47773c399cace71237ef53cdadc"}, - {file = "cvxpy-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d6bfbd535fdaabc5fa55f28de7a1d40f3a803a27fe3fec86e90700fa159a3afc"}, - {file = "cvxpy-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71a95aaccf22431fd25a63bcb12d583e1b0baeaeb4fafa3e25857cec03b9e2f3"}, - {file = "cvxpy-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d3bae3bf31e4eb6ed6407f78c6bc3c7bc4b4145cdbbb9ba8c61c3fc541d7067"}, - {file = "cvxpy-1.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41cfaecf86f85162ca53c7be7377b4143e316204fb9b6a7df8b7a08c826e3806"}, - {file = "cvxpy-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:edf66010e49b64d3f2dd1a7abde8fa3e615ce7a2b3eb185ab744b0beb3a6adb9"}, - {file = "cvxpy-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6b0f17dca85b2a410e73f5d84b28f35f57a20cfec1b0adc9b16f0f8aabff9961"}, - {file = "cvxpy-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9318c4e679b3db470e76e7f23cce362b038bd2d68c4a7326a7c21577ddbdc542"}, - {file = "cvxpy-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46ef722c8d1590875e86360d5781703dfcbd08be73eb98a2fc91a280870064"}, - {file = "cvxpy-1.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57593a852c563ce77bdb075a3e75f23d36d4b3162ebf3199b54cc7fe75088ef2"}, - {file = "cvxpy-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:db89b55025514bad821b1f1781bed373cbb6aa22fe84420431efd510dbe7f858"}, - {file = "cvxpy-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:372c0825cc6e6bb03ecc550d83718761a1bbdbbb48010fec6f9718581ebd45b5"}, - {file = "cvxpy-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:163caffd7f7f27b6cb151f4ccff283068e063c3673158793048761690cbe4bbe"}, - {file = "cvxpy-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f24067c54979b09910aea0a03256247121d8a8169538facf087c1923e9e2701a"}, - {file = "cvxpy-1.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a3ec054279880a9ebf5fd9d2ac4109acf944b8c45ea8b24e461680e34f3d7b5"}, - {file = "cvxpy-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:d220a7ee55907da9b55b98e5238d03735118d03b82855ba87b872cb2e6977367"}, - {file = "cvxpy-1.4.1.tar.gz", hash = "sha256:7a9ef34e3c57ff8c844d86f0a3834fb5575af19233947639de0ba577c6122e3e"}, -] - -[package.dependencies] -clarabel = ">=0.5.0" -ecos = ">=2" -numpy = ">=1.15" -osqp = ">=0.6.2" -pybind11 = "*" -scipy = ">=1.1.0" -scs = ">=3.0" - -[package.extras] -cbc = ["cylp (>=0.91.5)"] -clarabel = ["clarabel"] -cvxopt = ["cvxopt"] -diffcp = ["diffcp"] -glop = ["ortools (>=9.5,<9.8)"] -glpk = ["cvxopt"] -glpk-mi = ["cvxopt"] -gurobi = ["gurobipy"] -highs = ["scipy (>=1.6.1)"] -mosek = ["Mosek"] -pdlp = ["ortools (>=9.5,<9.8)"] -piqp = ["piqp"] -proxqp = ["proxsuite"] -scip = ["PySCIPOpt"] -scipy = ["scipy"] -scs = ["setuptools (>65.5.1)"] -xpress = ["xpress"] - [[package]] name = "cycler" version = "0.12.1" @@ -716,35 +637,6 @@ files = [ {file = "dm_tree-0.1.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368"}, ] -[[package]] -name = "ecos" -version = "2.0.12" -description = "This is the Python package for ECOS: Embedded Cone Solver. See Github page for more information." -optional = false -python-versions = "*" -files = [ - {file = "ecos-2.0.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:835298a299c88c207b3402fba60ad9b5688b59bbbf2ac34a46de5b37165d773a"}, - {file = "ecos-2.0.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608bc822ee8e070927ab3519169b13a1a0fe88f3d562212d6b5dbb1039776360"}, - {file = "ecos-2.0.12-cp310-cp310-win_amd64.whl", hash = "sha256:5184a9d8521ad1af90ffcd9902a6fa75c7bc473f37d30d86f97beda1033dfca2"}, - {file = "ecos-2.0.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:eba07599084724eedc20b2862d5580eebebb09609f4740baadc78401cb99827c"}, - {file = "ecos-2.0.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4979dc2d1cb6667e371a45a61887068505c1305437eef104ed6ef16f4b6aa0e3"}, - {file = "ecos-2.0.12-cp311-cp311-win_amd64.whl", hash = "sha256:da8fbbca3feb83a9e27075d29b3765417d0c80af8ea83cbdc4a558cae7b564af"}, - {file = "ecos-2.0.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f70e4547966f530fd7715756f7a65d5b9b90b312b9d37f243ef9356c05e7d74c"}, - {file = "ecos-2.0.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:617be25d74222849622b0f82b94a11abcf1fae78ccaf69977b328321ee6ffa0b"}, - {file = "ecos-2.0.12-cp37-cp37m-win_amd64.whl", hash = "sha256:29d00164eaea66ed54697a3b361c575284a8bca54f2623381a0635806c7303a7"}, - {file = "ecos-2.0.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4e86671397d1d2cd7cccff8a9c45be0541b0c60af8b92a0ff3581c9ed869db67"}, - {file = "ecos-2.0.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:858a4dd3177bdc8cc6e362031732f5177b62138a1e4ef91c0dc3c6bd7d2d1248"}, - {file = "ecos-2.0.12-cp38-cp38-win_amd64.whl", hash = "sha256:528b02f53835bd1baeb2e23f8153b8d6cc2b3704e1768be6a1a972f542241670"}, - {file = "ecos-2.0.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e42bd4c19af6e04f76ccc85d941b1f1adc7faeee4d06d482395a6beb7bec895"}, - {file = "ecos-2.0.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6def54336a15b5a49bc3bfcaa36035e8557cae8a4853b17ca84f5a29c93bcaea"}, - {file = "ecos-2.0.12-cp39-cp39-win_amd64.whl", hash = "sha256:7af08941552fce108bd80145cdb6be7fa74477a20bacdac170800442cc7027d4"}, - {file = "ecos-2.0.12.tar.gz", hash = "sha256:f48816d73b87ae325556ea537b7c8743187311403c80e3832035224156337c4e"}, -] - -[package.dependencies] -numpy = ">=1.6" -scipy = ">=0.9" - [[package]] name = "flatbuffers" version = "23.5.26" @@ -2278,6 +2170,17 @@ files = [ {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] +[[package]] +name = "nvidia-nccl-cu12" +version = "2.26.5" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = false +python-versions = ">=3" +files = [ + {file = "nvidia_nccl_cu12-2.26.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adb1bf4adcc5a47f597738a0700da6aef61f8ea4251b375540ae138c7d239588"}, + {file = "nvidia_nccl_cu12-2.26.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea5ed3e053c735f16809bee7111deac62ac35b10128a8c102960a0462ce16cbe"}, +] + [[package]] name = "oauthlib" version = "3.2.2" @@ -2312,29 +2215,6 @@ numpy = ">=1.7" docs = ["numpydoc", "sphinx (==1.2.3)", "sphinx-rtd-theme", "sphinxcontrib-napoleon"] tests = ["pytest", "pytest-cov", "pytest-pep8"] -[[package]] -name = "optbinning" -version = "0.18.0" -description = "OptBinning: The Python Optimal Binning library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "optbinning-0.18.0-py3-none-any.whl", hash = "sha256:9adf279051ac336ba81d2af456c26208e30d72fbc0c46b2df3fc43edad6d4745"}, - {file = "optbinning-0.18.0.tar.gz", hash = "sha256:0c05924c45a22a8f169f72da2371154d6d73e21e7576248272f9c9a63c7a04ab"}, -] - -[package.dependencies] -matplotlib = "*" -numpy = ">=1.16.1" -ortools = ">=9.4" -pandas = "*" -ropwr = ">=1.0.0" -scikit-learn = ">=1.0.2" -scipy = ">=1.6.0" - -[package.extras] -distributed = ["pympler", "tdigest"] - [[package]] name = "optuna" version = "3.5.0" @@ -2363,83 +2243,6 @@ integration = ["botorch (>=0.4.0)", "catboost (>=0.26)", "catboost (>=0.26,<1.2) optional = ["boto3", "botorch", "cmaes (>=0.10.0)", "google-cloud-storage", "matplotlib (!=3.6.0)", "pandas", "plotly (>=4.9.0)", "redis", "scikit-learn (>=0.24.2)"] test = ["coverage", "fakeredis[lua]", "kaleido", "moto", "pytest", "scipy (>=1.9.2)"] -[[package]] -name = "ortools" -version = "9.6.2534" -description = "Google OR-Tools python libraries and modules" -optional = false -python-versions = ">= 3.6" -files = [ - {file = "ortools-9.6.2534-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b2ed0a0d8691351905dbcef0246342a67c47f3b5d0401de12093fbd1b7554568"}, - {file = "ortools-9.6.2534-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05f0fab30582d183f5df86da8aa714327b488a1c51095aa8e58f1f57be58ecfe"}, - {file = "ortools-9.6.2534-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac7b95238571048e564502327a4c81300fa1225a7343d39ebd177502a3060ae4"}, - {file = "ortools-9.6.2534-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43e24847a089adf5e79b4dd26845f84d08b66013121123b2f9f5f90e452163bc"}, - {file = "ortools-9.6.2534-cp310-cp310-win_amd64.whl", hash = "sha256:25b4ad1e7c8fe58da27e14e0fc4a16b5c835cb00870a298f2e41d7d0eb13ab10"}, - {file = "ortools-9.6.2534-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:f4cdafb26f8ce9fc3149cb28c25bf84fa22bba807f91d938e0e68efe549a53c3"}, - {file = "ortools-9.6.2534-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:03b97413617fb7b288fa08f248c0e6d77e569da4f3a5f0a09e399d57faa6fbe6"}, - {file = "ortools-9.6.2534-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:886c2b7bfac12468f24486525acfc6e78a8a374d61916f50b15598bcf82daa0a"}, - {file = "ortools-9.6.2534-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d421138d4e4761324b3e2a7f4575969879e7c01d777fe554b616144eb743967"}, - {file = "ortools-9.6.2534-cp311-cp311-win_amd64.whl", hash = "sha256:fa65adff97e356acbf20a5c068595cfa247182621bb83671daa98359265f9927"}, - {file = "ortools-9.6.2534-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdb44dcd6fa66dad0fe4f2915700abb0f3ebb101983ad17e45098370f5d095ab"}, - {file = "ortools-9.6.2534-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eab4e658543d976936fbe4f79b87d6e1d66598ba6f6e2bca3e1a1471f897b072"}, - {file = "ortools-9.6.2534-cp37-cp37m-win_amd64.whl", hash = "sha256:0fe800f9823b342bf32917f9c7d770943eeac90641c24eeedd15eff600ae7aa8"}, - {file = "ortools-9.6.2534-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f961272381015c4d804c3077c19b5d80e64d7f69d8646e525fb27ed61afbacc9"}, - {file = "ortools-9.6.2534-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:19e43ff44331f4c2a4a5a886632ebe735c906acf6938aaac2813b37de806c948"}, - {file = "ortools-9.6.2534-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79479aec12c434af9cbe3f792e50fff6e84c1974bd7ee89c8148eea9459df064"}, - {file = "ortools-9.6.2534-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9661d68d3097f20ee7e789bbc0f7112d4afbd676a8a5dd94bbeda7002e22089a"}, - {file = "ortools-9.6.2534-cp38-cp38-win_amd64.whl", hash = "sha256:36ccc3e31af37ba767bdd0e08c5361f679e2cd2e61e25eab103dca8490a9dd54"}, - {file = "ortools-9.6.2534-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:76a27a0eb56e63a6981783e1e366725e8df684f126943c22960c86cfabc94697"}, - {file = "ortools-9.6.2534-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6bd17ea2c3bd58387a1fbc6bfa726e78dc6967ab8c798f56a12d9b6ebb37a1f9"}, - {file = "ortools-9.6.2534-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f62ece1909940a096e98544b552825335e34e8356b67fae972bc74609dd19c3d"}, - {file = "ortools-9.6.2534-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67159c5523c78900479979d9629f867d1f4fbe293b339aa84c6965019ae52210"}, - {file = "ortools-9.6.2534-cp39-cp39-win_amd64.whl", hash = "sha256:eb2c83163abf72cf74f17117dcc53a93004d3d35636c2511278349f7920d8c8d"}, -] - -[package.dependencies] -absl-py = ">=0.13" -numpy = ">=1.13.3" -protobuf = ">=4.21.12" -scipy = ">=1.10.0" - -[[package]] -name = "osqp" -version = "0.6.3" -description = "OSQP: The Operator Splitting QP Solver" -optional = false -python-versions = "*" -files = [ - {file = "osqp-0.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b7d923c836f1d07115057e595245ccc1694ecae730a1affda78fc6f3c8d239"}, - {file = "osqp-0.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dfda08c38c3521012740a73ef782f97dfc54a41deae4b0bc4afd18d0e74da0"}, - {file = "osqp-0.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7eafa3f3e82dd36c52f3f4ef19a95142405c807c272c4b53c5971c53535d7804"}, - {file = "osqp-0.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:3cbb6efdaffb7387dc0037dfe3259d4803e5ad7217e6f20fb605c92953214b9d"}, - {file = "osqp-0.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1b2049b2c42565dcaa63ddca1c4028b1fb20aab141453f5d77e8ff5b1a99a2cf"}, - {file = "osqp-0.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:146b89f2cfbf59eaeb2c47e3a312f2034138df78d80ce052364810dc0ef70fc4"}, - {file = "osqp-0.6.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0084e3d733c75687d68bc133bc380ce471dfe6f7724af2718a43491782eec8d6"}, - {file = "osqp-0.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:1b573fe1cd0e82239a279c58817c1d365187ef862e928b2b9c828c3c516ad3c2"}, - {file = "osqp-0.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c3951ef505177b858c6cd34de980346014cae3d2234c93db960b12c5885f9a2"}, - {file = "osqp-0.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc18f87c9549032c163ce590a5e32079df94ee656c8fb357ba607aa9d78fab81"}, - {file = "osqp-0.6.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c07b1a4b538aab629b0fae69f644b7e76f81f94d65230014d482e296dacd046b"}, - {file = "osqp-0.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:60abec3593870990b16f00bd5017096a7091fb00b68d0db3383fc048ca8e55c9"}, - {file = "osqp-0.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b73bdd9589901841af83c5ed6a4092b4fac5a0beff9e32682d8526d1f16a728c"}, - {file = "osqp-0.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71d9f611823af4a8b241c86805920e5382cd65c7f94fd3615b4eef999ed94c7c"}, - {file = "osqp-0.6.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30fbc3b3c028c06a6c5f1e66be7b7106ad48a29e0dc5bd82393f82dd68235ef8"}, - {file = "osqp-0.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fe57e4bde071b388518ecb068f26319506dd9cb107363d3d80c12d2e59fc1e81"}, - {file = "osqp-0.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41f304d1d7f91af07d8f0b01e5af29ec3bb8824f0102c7fd8b13b497be120da4"}, - {file = "osqp-0.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea7d8c92bcdf4fef98d777f13d39060d425ef2e8778ed487c96a6fa10848cdea"}, - {file = "osqp-0.6.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f3a3c6d2708868e5e3fe2da300d6523cbf68a3d8734ce9c5043db37391969f5"}, - {file = "osqp-0.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:1c548a0b3691850e7e22f3624a128d8af33416d70a9b5976a47d4d832028dcd8"}, - {file = "osqp-0.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:387e7abd737dfe32c9ec00ad74af25328cdd0d0f634d79530655c040a5cb9590"}, - {file = "osqp-0.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1445e10a94e01698e13c87a7debf6ac1a15f3acd1f8f6340cb1ad945db4732b"}, - {file = "osqp-0.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0441c10f7fe5f46692a9b44a57138977bb112ae3f8127151671968c5d9ec5dbb"}, - {file = "osqp-0.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:b15e65a307fbbabf60248bb9bc204e61d5d4ae64e00427a69e2dad9622f4c29d"}, - {file = "osqp-0.6.3.tar.gz", hash = "sha256:03e460e683ec2ce0f839353ddfa3c4c8ffa509ab8cf6a2b2afbb586fa453e180"}, -] - -[package.dependencies] -numpy = ">=1.7" -qdldl = "*" -scipy = ">=0.13.2" - [[package]] name = "packaging" version = "23.2" @@ -2859,20 +2662,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.6.0" -[[package]] -name = "pybind11" -version = "2.11.1" -description = "Seamless operability between C++11 and Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pybind11-2.11.1-py3-none-any.whl", hash = "sha256:33cdd02a6453380dd71cc70357ce388ad1ee8d32bd0e38fc22b273d050aa29b3"}, - {file = "pybind11-2.11.1.tar.gz", hash = "sha256:00cd59116a6e8155aecd9174f37ba299d1d397ed4a6b86ac1dfe01b3e40f2cc4"}, -] - -[package.extras] -global = ["pybind11-global (==2.11.1)"] - [[package]] name = "pydantic" version = "2.5.3" @@ -3154,38 +2943,6 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] -[[package]] -name = "qdldl" -version = "0.1.7.post0" -description = "QDLDL, a free LDL factorization routine." -optional = false -python-versions = "*" -files = [ - {file = "qdldl-0.1.7.post0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ab02e8b9ff86bd644a1935718387c82fbe04c31e3309cf9f7a121d02b1deda8"}, - {file = "qdldl-0.1.7.post0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e5d6753310377451ed4dc09b1ef28faf40108b713e7f55c8a8ae94d679a672"}, - {file = "qdldl-0.1.7.post0-cp310-cp310-win_amd64.whl", hash = "sha256:718d8e141832e96ba71ca1807a74813836c6403110faaa3d33a67de1af3b29c4"}, - {file = "qdldl-0.1.7.post0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e3f06e8a49ddd834b24fc3d7afbba4fec0923101045aa2666e18d2a9980e329"}, - {file = "qdldl-0.1.7.post0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a81c46522dd6b3042e2348fa98128bb5c0e466f42bce214e80cfb766ff40930"}, - {file = "qdldl-0.1.7.post0-cp311-cp311-win_amd64.whl", hash = "sha256:4a86155f3de66c5db0e21544b7a2421c671028fa20da407686d2a8d0e9b57e51"}, - {file = "qdldl-0.1.7.post0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:717cb1892b033c01a0aae84ededcfa1f05bcb97013095d779c497e6c32f90dac"}, - {file = "qdldl-0.1.7.post0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fc35432913085d94b2327242cf51388467ef7a37ac0d71eb31b594b575dd498"}, - {file = "qdldl-0.1.7.post0-cp36-cp36m-win_amd64.whl", hash = "sha256:fd5cfd8c50f33ddacb830594a63b8c1093a24aea45312b9d2ed826cea5ece08a"}, - {file = "qdldl-0.1.7.post0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:981ca8672e9506976c663552c1eb6f6daf9726d62650b3bf5900260946156166"}, - {file = "qdldl-0.1.7.post0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8ec670d97cf756f9159dc0a11de5cf054e88aefe84bea1c7282f00334642843"}, - {file = "qdldl-0.1.7.post0-cp37-cp37m-win_amd64.whl", hash = "sha256:aa208703b44337a7e77f6f2663f7a452144becb4421970d534ff8297b92e1e10"}, - {file = "qdldl-0.1.7.post0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b42649484f7c0d8ee659224ecaac0a3e97f12531018207f4d7323e4071320eb1"}, - {file = "qdldl-0.1.7.post0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26aa3d6f0da7779265d72e8f418094003e75fa53c515a53bc03fd8b9bcfbf7de"}, - {file = "qdldl-0.1.7.post0-cp38-cp38-win_amd64.whl", hash = "sha256:e55bcd6962178029faf543addd49db145302dd51e19855fefa71b5fd55840eea"}, - {file = "qdldl-0.1.7.post0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c1dd0e570e65aaf35e10b7fb345f7ac763fd05a2227b9c06ce65e07993fc4984"}, - {file = "qdldl-0.1.7.post0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae161342529852b6248ace4642bc4ee371a7c1e0707b7bc43a43ef7e73c06ca3"}, - {file = "qdldl-0.1.7.post0-cp39-cp39-win_amd64.whl", hash = "sha256:092f6606690a2b9bd3c939f3147887e02de13bb068fbed5ffdc7459034def623"}, - {file = "qdldl-0.1.7.post0.tar.gz", hash = "sha256:f346a114c8342ee6d4dbd6471eef314199fb268d3bf7b95885ca351fde2b023f"}, -] - -[package.dependencies] -numpy = ">=1.7" -scipy = ">=0.13.2" - [[package]] name = "referencing" version = "0.32.1" @@ -3240,23 +2997,6 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] -[[package]] -name = "ropwr" -version = "1.0.0" -description = "RoPWR: Robust Piecewise Regression" -optional = false -python-versions = ">=3.7" -files = [ - {file = "ropwr-1.0.0-py3-none-any.whl", hash = "sha256:7ecedd09e513e7fc2ef15b2c236fd25248ce73b0fb7642b9fab6c68d05749de5"}, - {file = "ropwr-1.0.0.tar.gz", hash = "sha256:723981924e57d0601a753a25d3d5d925da7c43d169d55f400cb00970b73e3605"}, -] - -[package.dependencies] -cvxpy = ">=1.1.14" -numpy = ">=1.16" -scikit-learn = ">=0.22" -scipy = ">=1.6.1" - [[package]] name = "rpds-py" version = "0.17.1" @@ -3488,39 +3228,6 @@ dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyl doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] -[[package]] -name = "scs" -version = "3.2.4.post1" -description = "Splitting conic solver" -optional = false -python-versions = ">=3.7" -files = [ - {file = "scs-3.2.4.post1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51fed30d2a4a1e6fbfc1e52b4cb3adeecbe89d7c47f3539b49afbb852415fe19"}, - {file = "scs-3.2.4.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb0524c0b9c3ed0d65dae161475accf3efa8e170938eb93251a60e9709b156ee"}, - {file = "scs-3.2.4.post1-cp310-cp310-win_amd64.whl", hash = "sha256:534519819eea96f18902a9fce15c4ec562b99d23b38dc843a48cb137b5641613"}, - {file = "scs-3.2.4.post1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d04ee4d19ac6d0f5053663bc48fcd5c5faed534272f13b10a4e173c814eea69"}, - {file = "scs-3.2.4.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c23b4299ab77ff5f654573d5667dc982292a8ef2b979053b38c40663919f13"}, - {file = "scs-3.2.4.post1-cp311-cp311-win_amd64.whl", hash = "sha256:ae4624938d3e3a8b7e508029275c6ad7a978fd48c158d0818f69f4ae764bf945"}, - {file = "scs-3.2.4.post1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:40294e22bfe509bdf7fd65a6b77c38cec22dcb3567ff5a75f3c41a1faf2ef1d5"}, - {file = "scs-3.2.4.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a2337acb0604770b6df1473254065a51c210ff9c82fc7c4490595510287a337"}, - {file = "scs-3.2.4.post1-cp312-cp312-win_amd64.whl", hash = "sha256:8689e75a57e59846e65d1c4b9d57e9964b00fcbb8e67fc77f98cf6e0a0530abd"}, - {file = "scs-3.2.4.post1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ad991b00d0a87c85db57bf2f1863c21bdc4e2f13837f6c35e809f5936bc6f165"}, - {file = "scs-3.2.4.post1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a28af160a44268e726a59d6cf340629b82940c1a643c4c87fe777e9cbe550d75"}, - {file = "scs-3.2.4.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:f6283f725f3fee63d4631c2532d01a5b2ea65883b04d3da3be06084b1c60171b"}, - {file = "scs-3.2.4.post1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b3a622cf2120ae765f0f3ad5c6f4f86796d317e29132bab2ad4af3c14d9bf4d"}, - {file = "scs-3.2.4.post1-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:4b5259137c263304effa2b28d0125437ac23569e6e7753c115ae1206ec5033fd"}, - {file = "scs-3.2.4.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424710bc19b0506feee7e05e6d2b7af98acf09af5bd5353126164cbd46ac923f"}, - {file = "scs-3.2.4.post1-cp38-cp38-win_amd64.whl", hash = "sha256:e21bdc8046648846e2c204a6c5cf24eaaedd2b8f5e0a2ab41a647b0247b8d592"}, - {file = "scs-3.2.4.post1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cea0f7e9473f43f7edf1641d020ead7e39653a81c540fbdba8f3b7b8480038c9"}, - {file = "scs-3.2.4.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6126f1d7ed5ff368cb8c1836715b17a50074314579eefc6d511995a3ab93d70"}, - {file = "scs-3.2.4.post1-cp39-cp39-win_amd64.whl", hash = "sha256:18788befa5284bb1f49149bac7f813703de60ef5b6bf7698a9f1c3a5a49b78e4"}, - {file = "scs-3.2.4.post1.tar.gz", hash = "sha256:7015d7a56d1d5b53264fd277289ea169949309e26101677ff88cd0e5030d032f"}, -] - -[package.dependencies] -numpy = "*" -scipy = "*" - [[package]] name = "setuptools" version = "69.0.3" @@ -4042,6 +3749,35 @@ io = ["cftime", "fsspec", "h5netcdf", "netCDF4", "pooch", "pydap", "scipy", "zar parallel = ["dask[complete]"] viz = ["matplotlib", "nc-time-axis", "seaborn"] +[[package]] +name = "xgboost" +version = "3.0.0" +description = "XGBoost Python Package" +optional = false +python-versions = ">=3.10" +files = [ + {file = "xgboost-3.0.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:ed8cffd7998bd9431c3b0287a70bec8e45c09b43c9474d9dfd261627713bd890"}, + {file = "xgboost-3.0.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:314104bd3a1426a40f0c9662eef40e9ab22eb7a8068a42a8d198ce40412db75c"}, + {file = "xgboost-3.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:72c3405e8dfc37048f9fe339a058fa12b9f0f03bc31d3e56f0887eed2ed2baa1"}, + {file = "xgboost-3.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:72d39e74649e9b628c4221111aa6a8caa860f2e853b25480424403ee61085126"}, + {file = "xgboost-3.0.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:7bdee5787f86b83bebd75e2c96caf854760788e5f4203d063da50db5bf0efc5f"}, + {file = "xgboost-3.0.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:61c7e391e373b8a5312503525c0689f83ef1912a1236377022865ab340f465a4"}, + {file = "xgboost-3.0.0-py3-none-win_amd64.whl", hash = "sha256:0ea74e97f95b1eddfd27a46b7f22f72ec5a5322e1dc7cb41c9c23fb580763df9"}, + {file = "xgboost-3.0.0.tar.gz", hash = "sha256:45e95416df6f6f01d9a62e60cf09fc57e5ee34697f3858337c796fac9ce3b9ed"}, +] + +[package.dependencies] +numpy = "*" +nvidia-nccl-cu12 = {version = "*", markers = "platform_system == \"Linux\" and platform_machine != \"aarch64\""} +scipy = "*" + +[package.extras] +dask = ["dask", "distributed", "pandas"] +pandas = ["pandas (>=1.2)"] +plotting = ["graphviz", "matplotlib"] +pyspark = ["cloudpickle", "pyspark", "scikit-learn"] +scikit-learn = ["scikit-learn"] + [[package]] name = "yarl" version = "1.9.4" @@ -4148,4 +3884,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "3a2ee6dbc0ee725a523d9baa7dede0a194cdada5b1a169873ca2471b8b9f1a15" +content-hash = "18673fa30afeea35ab6e166d0272595194e6e6c0ae521cfd9ae8b6627d71eb53" diff --git a/jupyterhub/pyproject.toml b/jupyterhub/pyproject.toml index 072baae..e92d945 100644 --- a/jupyterhub/pyproject.toml +++ b/jupyterhub/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "jupyterhub" -version = "2.0.0" +version = "2.0.1" description = "" authors = ["Abhinav "] readme = "README.md" @@ -26,6 +26,7 @@ influxdb = "^5.3.1" google-cloud-bigquery = "^3.16.0" tensorflow = "2.14.0" tensorflow-probability = "^0.23.0" +xgboost = "^3.0.0" [build-system] diff --git a/jupyterhub/requirements.txt b/jupyterhub/requirements.txt index 83bc8f1..183559d 100644 --- a/jupyterhub/requirements.txt +++ b/jupyterhub/requirements.txt @@ -68,6 +68,7 @@ mysql-connector-python==8.2.0 ; python_version >= "3.11" and python_version < "3 numba==0.58.1 ; python_version >= "3.11" and python_version < "3.12" numexpr==2.8.8 ; python_version >= "3.11" and python_version < "3.12" numpy==1.26.3 ; python_version >= "3.11" and python_version < "3.12" +nvidia-nccl-cu12==2.26.5 ; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" and python_version < "3.12" oauthlib==3.2.2 ; python_version >= "3.11" and python_version < "3.12" opt-einsum==3.3.0 ; python_version >= "3.11" and python_version < "3.12" optuna==3.5.0 ; python_version >= "3.11" and python_version < "3.12" @@ -123,4 +124,5 @@ werkzeug==3.0.1 ; python_version >= "3.11" and python_version < "3.12" wheel==0.42.0 ; python_version >= "3.11" and python_version < "3.12" wrapt==1.14.1 ; python_version >= "3.11" and python_version < "3.12" xarray==2023.12.0 ; python_version >= "3.11" and python_version < "3.12" +xgboost==3.0.0 ; python_version >= "3.11" and python_version < "3.12" yarl==1.9.4 ; python_version >= "3.11" and python_version < "3.12" diff --git a/pyspark/executors/Dockerfile b/pyspark/executors/Dockerfile index 8c87616..a96ab8c 100644 --- a/pyspark/executors/Dockerfile +++ b/pyspark/executors/Dockerfile @@ -1,5 +1,6 @@ -FROM us-docker.pkg.dev/vgi-pn-277619/data-team/pubnative/pyspark:v3.4.1-python3.11-2 +#FROM us-docker.pkg.dev/vgi-pn-277619/data-team/pubnative/pyspark:v3.4.1-python3.11-2 # use the base image here +FROM us-docker.pkg.dev/vgi-pn-277619/data-team/pubnative/pyspark-executor:v3.4.1-python3.11-2 USER 0 diff --git a/pyspark/executors/Makefile b/pyspark/executors/Makefile index 6098734..e51efe9 100644 --- a/pyspark/executors/Makefile +++ b/pyspark/executors/Makefile @@ -1,6 +1,6 @@ OWNER := us-docker.pkg.dev/vgi-pn-277619/data-team/pubnative REPO := ${OWNER}/pyspark-executor -TAG := v3.4.1-python3.11-2 +TAG := v3.4.1-python3.11-3 IMAGE := ${REPO}:${TAG} build: @@ -8,3 +8,6 @@ build: push: build docker push ${IMAGE} + +requirements: + cp ../../jupyterhub/requirements.txt . \ No newline at end of file diff --git a/pyspark/executors/requirements.txt b/pyspark/executors/requirements.txt index ab4120f..7d0351f 100644 --- a/pyspark/executors/requirements.txt +++ b/pyspark/executors/requirements.txt @@ -68,6 +68,7 @@ mysql-connector-python==8.2.0 ; python_version >= "3.11" and python_version < "3 numba==0.58.1 ; python_version >= "3.11" and python_version < "3.12" numexpr==2.8.8 ; python_version >= "3.11" and python_version < "3.12" numpy==1.26.3 ; python_version >= "3.11" and python_version < "3.12" +nvidia-nccl-cu12==2.26.5 ; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" and python_version < "3.12" oauthlib==3.2.2 ; python_version >= "3.11" and python_version < "3.12" opt-einsum==3.3.0 ; python_version >= "3.11" and python_version < "3.12" optuna==3.5.0 ; python_version >= "3.11" and python_version < "3.12" @@ -89,6 +90,7 @@ pydantic-core==2.14.6 ; python_version >= "3.11" and python_version < "3.12" pydantic==2.5.3 ; python_version >= "3.11" and python_version < "3.12" pydata-google-auth==1.8.2 ; python_version >= "3.11" and python_version < "3.12" pyparsing==3.1.1 ; python_version >= "3.11" and python_version < "3.12" +pyspark==3.4.1 ; python_version >= "3.11" and python_version < "3.12" python-dateutil==2.8.2 ; python_version >= "3.11" and python_version < "3.12" pytz==2023.3.post1 ; python_version >= "3.11" and python_version < "3.12" pytzdata==2020.1 ; python_version >= "3.11" and python_version < "3.12" @@ -121,4 +123,5 @@ werkzeug==3.0.1 ; python_version >= "3.11" and python_version < "3.12" wheel==0.42.0 ; python_version >= "3.11" and python_version < "3.12" wrapt==1.14.1 ; python_version >= "3.11" and python_version < "3.12" xarray==2023.12.0 ; python_version >= "3.11" and python_version < "3.12" +xgboost==3.0.0 ; python_version >= "3.11" and python_version < "3.12" yarl==1.9.4 ; python_version >= "3.11" and python_version < "3.12"