diff --git a/.github/workflows/auto-accept-ci-changes.yml b/.github/workflows/auto-accept-ci-changes.yml index 3c9a34d8..4908e999 100644 --- a/.github/workflows/auto-accept-ci-changes.yml +++ b/.github/workflows/auto-accept-ci-changes.yml @@ -33,6 +33,8 @@ jobs: uses: dependabot/fetch-metadata@ffa630c65fa7e0ecfa0625b5ceda64399aea1b36 # v3.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} + permission-contents: write + permission-pull-requests: write - name: Stop workflow if not minor update or patch update id: skip-condition diff --git a/.github/workflows/cache_cleaner.yml b/.github/workflows/cache_cleaner.yml index 7403251d..78486309 100644 --- a/.github/workflows/cache_cleaner.yml +++ b/.github/workflows/cache_cleaner.yml @@ -27,6 +27,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Cleanup run: | diff --git a/.github/workflows/first_pull_request.yml b/.github/workflows/first_pull_request.yml index 50a52446..c99a3a11 100644 --- a/.github/workflows/first_pull_request.yml +++ b/.github/workflows/first_pull_request.yml @@ -20,6 +20,7 @@ jobs: with: disable-sudo: true egress-policy: audit + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ca44ee4..0bc9f058 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,24 +35,29 @@ jobs: files.pythonhosted.org:443 github.com:443 pypi.org:443 + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Python${{ matrix.python-version }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} cache: pip + - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Caching uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | .tox key: ${{ runner.os }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-lint-${{ env.ESGF_TEST_DATA_VERSION }} + - name: Run linting suite run: | python -m tox -e lint @@ -71,22 +76,27 @@ jobs: with: disable-sudo: false egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Python${{ matrix.python-version }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} cache: "pip" + - name: Install HDF5 (Linux) run: | sudo apt-get update sudo apt-get install -y libhdf5-dev + - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Caching uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: @@ -95,6 +105,7 @@ jobs: ~/.cache/xclim-testdata .tox key: ${{ runner.os }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-Python${{ matrix.python-version }}-${{ env.ESGF_TEST_DATA_VERSION }} + - name: Test with tox run: | python -m tox @@ -112,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.11", "3.12", "3.13" ] + python-version: [ "3.11", "3.12", "3.13", "3.14" ] defaults: run: shell: bash -l {0} @@ -122,10 +133,12 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0 with: @@ -134,10 +147,11 @@ jobs: environment-file: environment.yml create-args: >- python=${{ matrix.python-version }} - pytest-timeout + - name: Install CLISOPS run: | python -m pip install --no-user --no-deps --editable . + - name: Test Data Caching uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: @@ -145,13 +159,16 @@ jobs: ~/.cache/mini-esgf-data ~/.cache/xclim-testdata key: ${{ runner.os }}-${{ hashFiles('pyproject.toml', 'tox.ini') }}-conda-Python${{ matrix.python-version }}-${{ env.ESGF_TEST_DATA_VERSION }} + - name: Check versions run: | micromamba list python -m pip check || true + - name: Test with conda run: | - python -m pytest -m "not slow" --timeout=300 --numprocesses=logical --durations=10 --cov=clisops --cov-report=lcov + python -m pytest -m "not slow" --numprocesses=logical --durations=10 --cov=clisops --cov-report=lcov + - name: Report Coverage uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 with: @@ -170,6 +187,7 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Coveralls Finished uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 with: diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 64d3a1f0..5044f1ff 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -21,10 +21,12 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Create Release uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1 env: diff --git a/.github/workflows/publish-pypi-production.yml b/.github/workflows/publish-pypi-production.yml index 5fd40d30..ae4afca4 100644 --- a/.github/workflows/publish-pypi-production.yml +++ b/.github/workflows/publish-pypi-production.yml @@ -25,19 +25,24 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Python${{ matrix.python-version }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "${{ matrix.python-version }}" + - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Build a binary wheel and a source tarball run: | python -m flit build + - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/tag-testpypi-staging.yml b/.github/workflows/tag-testpypi-staging.yml index 4a3cb289..71e8cc56 100644 --- a/.github/workflows/tag-testpypi-staging.yml +++ b/.github/workflows/tag-testpypi-staging.yml @@ -25,20 +25,25 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Set up Python${{ matrix.python-version }} uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "${{ matrix.python-version }}" + - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Build a binary wheel and a source tarball run: | python -m flit build + - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 5f69af74..9cb9162d 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -42,11 +42,13 @@ jobs: with: disable-sudo: true egress-policy: audit + - name: Checkout Repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Fetch all history for all branches and tags. persist-credentials: false + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0 with: @@ -56,21 +58,26 @@ jobs: create-args: >- python=${{ matrix.python-version }} pytest-reportlog + - name: Install CLISOPS run: | python -m pip install --no-deps --no-user --editable . + - name: Install upstream versions run: | python -m pip install -r CI/requirements_upstream.txt + - name: Check versions run: | micromamba list python -m pip check || true + - name: Run Tests if: success() id: status run: | python -m pytest --durations=10 --cov=clisops --cov-report=term-missing --report-log output-${{ matrix.python-version }}-log.jsonl + - name: Generate and publish the report if: | failure() diff --git a/.gitignore b/.gitignore index c7348c05..00bf926c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,7 @@ htmlcov/ .coverage.* .cache nosetests.xml -coverage.xml +coverage.* *.cover .hypothesis/ .pytest_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99f2058f..d1e1a94a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,17 +10,21 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - - id: end-of-file-fixer - - id: fix-byte-order-marker - id: check-merge-conflict + - id: check-json - id: check-toml - id: check-yaml - id: debug-statements + - id: end-of-file-fixer + - id: fix-byte-order-marker - id: mixed-line-ending - - id: no-commit-to-branch - args: [ '--branch', 'main' ] - id: name-tests-test args: [ '--pytest-test-first' ] + - id: no-commit-to-branch + args: [ '--branch', 'main' ] + - id: pretty-format-json + args: [ '--autofix', '--no-ensure-ascii', '--no-sort-keys' ] + exclude: .ipynb - id: trailing-whitespace - repo: https://github.com/pappasam/toml-sort rev: v0.24.4 @@ -32,7 +36,7 @@ repos: - id: yamllint args: [ '--config-file=.yamllint.yaml' ] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.9 + rev: v0.15.15 hooks: - id: ruff-check args: [ '--fix', '--show-fixes' ] @@ -69,11 +73,11 @@ repos: - id: numpydoc-validation exclude: ^docs/|^tests/ - repo: https://github.com/gitleaks/gitleaks - rev: v8.30.0 + rev: v8.30.1 hooks: - id: gitleaks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.37.1 + rev: 0.37.2 hooks: - id: check-github-workflows - id: check-readthedocs diff --git a/.readthedocs.yml b/.readthedocs.yml index 1d12bd33..d6e0ba8a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,9 +12,9 @@ sphinx: # Build configuration build: - os: ubuntu-24.04 + os: "ubuntu-26.04" tools: - python: "mambaforge-23.11" + python: "miniforge3-25.11" jobs: post_create_environment: - mamba install --quiet --name ${READTHEDOCS_VERSION} -c conda-forge "python>=3.12,<3.13" "psy-maps>=1.5.0" diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0bec879b..be67007a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -86,34 +86,35 @@ Ready to contribute? Here's how to set up `clisops` for local development. Now you can make your changes locally! #. - When you are done making changes, check that you verify your changes with `flake8` and `black` and run the tests, including testing other Python versions with `tox`: + When you are done making changes, check that you verify your changes with `deptry`, `flake8`, and `ruff` and run the tests, including testing other Python versions with `tox`: .. code-block:: shell # For virtualenv environments: - $ pip install flake8 black pytest pytest-loguru tox + $ pip install flake8 ruff deptry pytest pytest-loguru tox # For Anaconda/Miniconda environments: $ conda install -c conda-forge flake8 black pytest pytest-loguru tox - $ flake8 clisops tests - $ black clisops tests + $ ruff check clisops tests + $ flake8 --config=.flake8 clisops tests + $ deptry . $ pytest $ tox #. - Before committing your changes, we ask that you install `pre-commit` in your virtualenv. `Pre-commit` runs git hooks that ensure that your code resembles that of the project and catches and corrects any small errors or inconsistencies when you `git commit`: + Before committing your changes, we ask that you install `prek` in your virtualenv. `prek` runs git hooks that ensure that your code resembles that of the project and catches and corrects any small errors or inconsistencies when you `git commit`: .. code-block:: shell # For virtualenv environments: - $ pip install pre-commit + $ pip install prek # For Anaconda/Miniconda environments: - $ conda install -c conda-forge pre-commit + $ conda install -c conda-forge prek - $ pre-commit install - $ pre-commit run --all-files + $ prek install + $ prek run --all-files #. Commit your changes and push your branch to GitHub: diff --git a/HISTORY.rst b/HISTORY.rst index 3c2fb9fa..1cc883e9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,14 +1,40 @@ Version History =============== - -v0.17.0 (2025-12-16) +v0.18.0 (unreleased) -------------------- New Features ^^^^^^^^^^^^ * Added support for a data mask in `subset_gridpoint` where the subsetted grid points but be within the mask (True) (#493). * Allows choice between using true world distance (`distance`) or nearest neighbour based on lat, lon (`geographic`) methods for subsetting both regular and irregular grids (#493). + +Bug Fixes +^^^^^^^^^ +* The `regrid.ipynb` notebook had malformed Python examples that were causing issues when building with `sphinx-codeautolink`. The issues have been addressed. (#498). + +Breaking Changes +^^^^^^^^^^^^^^^^ +* Default method for `subset_gridpoint` using regular lat,lon grids is now `distance` instead of previously employing the equivalent of the new `geographic` method (#493). +* Many development dependencies have been updated to more modern versions; `pytest` (v9.0+), `sphinx` (v8.2+), `matplotlib` (v3.8+), `tox` (v4.52.0+) and others. (#498). +* The pin on `numpy` (`<2.3.0`) has been lifted. (#498). + +Internal Changes +^^^^^^^^^^^^^^^^ +* Added a workflow to automatically accept patch and minor updates to CI dependencies via Dependabot. (#479). +* Adjusted GitHub Workflows security by constraining token and job permissions to stricter defaults. (#498). +* `pre-commit` has been replaced by `prek`. `prek` is a Rust-built reimplementation of `pre-commit` that runs very fast and is low on resources. (#498). +* `sphinx-autobuild` is now a development dependency, used for the `$ make servedocs` recipe (`make livehtml`). (#498). +* `pytest-timeout` is now a development dependency, used to prevent stalled tests from freezing CI builds (timeout default: 300 seconds). (#498). +* `pretty-format-json` hook for `pre-commit` has been enabled. (#498). +* ReadTheDocs build images have been updated to modern versions: OS `ubuntu-26.04` and Python `miniforge3-25.11`. (#498). +* Tests marked "outdated" have been removed from the testing suite. (#498). + +v0.17.0 (2025-12-16) +-------------------- + +New Features +^^^^^^^^^^^^ * Added an `engine` argument to `Grid.ds.to_netcdf()` to allow users to specify the engine used for writing NetCDF files (#439). * Coding conventions have been updated to use Python 3.10+ features (#439). * `Weights` will now use `post_mask_source='domain_edge'` introduced in `xesmf` version 0.9 when remapping a regional grid via nearest-neighbour to avoid extrapolation beyond the source domain (#447). @@ -27,7 +53,6 @@ Bug Fixes Breaking Changes ^^^^^^^^^^^^^^^^ -* Default method for `subset_gridpoint` using regular lat,lon grids is now `distance` instead of previously employing the equivalent of the new `geographic` method (#493). * Support for Python 3.10 has been dropped. `numpy >=1.26` is the new minimum supported version (#469). * `Grid.detect_extent()` now returns a tuple `(lon_extent, lat_extent)` instead of only `lon_extent` (#447). * `Grid.extent` now represents the combined lon/lat extent: `"global"` if both are global; otherwise `"regional"`. The new `Grid.extent_lon` and `Grid.extent_lat` attributes provide axis-specific extent information (#447). diff --git a/Makefile b/Makefile index 38e54642..ad50de21 100644 --- a/Makefile +++ b/Makefile @@ -78,14 +78,16 @@ coverage: ## check code coverage quickly with the default Python autodoc: clean-docs ## create sphinx-apidoc files: env SPHINX_APIDOC_OPTIONS="members,undoc-members,show-inheritance,no-index" sphinx-apidoc -o docs/apidoc --private --module-first clisops -docs: autodoc ## generate Sphinx HTML documentation, including API docs +build-docs: autodoc ## generate Sphinx HTML documentation, including API docs $(MAKE) -C docs html + +docs: build-docs ## open the built documentation in a web browser ifndef READTHEDOCS $(BROWSER) docs/_build/html/index.html endif -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.md' -c '$(MAKE) -C docs html' -R -D . +servedocs: autodoc ## compile the docs watching for changes + $(MAKE) -C docs livehtml dist: clean ## builds source and wheel package python -m flit build @@ -95,10 +97,11 @@ release: dist ## package and upload a release python -m flit publish dist/* install: clean ## install the package to the active Python's site-packages - python -m flit install + python -m pip install develop: clean ## install the package and development dependencies in editable mode to the active Python's site-packages - python -m flit install --no-user --symlink + python -m pip install --no-user --editable ".[dev]"" + prek install upstream: develop ## install the GitHub-based development branches of dependencies in editable mode to the active Python's site-packages python -m pip install --no-user --requirement requirements_upstream.txt diff --git a/README.rst b/README.rst index f585a383..74e6994d 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ clisops - climate simulation operations +----------------------------+---------------------------------+ | Open Source | |license| | +----------------------------+---------------------------------+ -| Coding Standards | |ruff| |pre-commit| | +| Coding Standards | |ruff| |prek| |pre-commit-ci| | +----------------------------+---------------------------------+ | Development Status | |status| |build| |coveralls| | +----------------------------+---------------------------------+ @@ -87,10 +87,14 @@ This package was created with ``Cookiecutter`` and the ``audreyr/cookiecutter-py :target: https://github.com/roocs/clisops/blob/master/LICENSE :alt: License -.. |pre-commit| image:: https://results.pre-commit.ci/badge/github/roocs/clisops/master.svg +.. |pre-commit-ci| image:: https://results.pre-commit.ci/badge/github/roocs/clisops/master.svg :target: https://results.pre-commit.ci/latest/github/roocs/clisops/master :alt: pre-commit.ci status +.. |prek| image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/j178/prek/master/docs/assets/badge-v0.json + :target: https://github.com/j178/prek + :alt: prek + .. |pypi| image:: https://img.shields.io/pypi/v/clisops.svg :target: https://pypi.python.org/pypi/clisops :alt: Python Package Index Build diff --git a/docs/Makefile b/docs/Makefile index 1d231a51..91e1c009 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -18,3 +18,7 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + + +livehtml: + sphinx-autobuild --port 54345 --open-browser --delay 3 --re-ignore "$(BUILDDIR)|apidoc|Makefile|__pycache__" "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index dea76e64..0c593ed7 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,7 +47,7 @@ "nbsphinx", # sphinx_autodoc_typehints must always be listed after sphinx.ext.napoleon "sphinx_autodoc_typehints", - # "sphinx_codeautolink", + "sphinx_codeautolink", "sphinx_copybutton", "IPython.sphinxext.ipython_console_highlighting", ] diff --git a/docs/index.rst b/docs/index.rst index ae52784b..f316ccb0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,8 +21,15 @@ Welcome to clisops's documentation! apidoc/modules -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +.. toctree:: + :caption: GitHub Repository + + roocs/clisops + +.. only:: html + + Indices and tables + ================== + * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` diff --git a/docs/notebooks/regrid.ipynb b/docs/notebooks/regrid.ipynb index bdb9886b..965d1857 100644 --- a/docs/notebooks/regrid.ipynb +++ b/docs/notebooks/regrid.ipynb @@ -86,9 +86,7 @@ " *,\n", " method: Optional[str] = \"nearest_s2d\",\n", " adaptive_masking_threshold: Optional[Union[int, float]] = 0.5,\n", - " grid: Optional[\n", - " Union[xr.Dataset, xr.DataArray, int, float, tuple, str]\n", - " ] = \"adaptive\",\n", + " grid: Optional[Union[xr.Dataset, xr.DataArray, int, float, tuple, str]] = \"adaptive\",\n", " output_dir: Optional[Union[str, Path]] = None,\n", " output_type: Optional[str] = \"netcdf\",\n", " split_method: Optional[str] = \"time:auto\",\n", @@ -979,6 +977,7 @@ " adaptive_masking_threshold: Optional[float] = 0.5,\n", " keep_attrs: Optional[bool] = True,\n", "):\n", + " pass\n", "```\n", "\n", "* `grid_in` and `grid_out` are `Grid` objects, `weights` is a `Weights` object.\n", @@ -988,7 +987,6 @@ " * `False` : The resulting `xarray.Dataset` will have no attributes despite attributes generated by the regridding process.\n", " * `\"target\"` : The resulting `xarray.Dataset` will have all attributes of `grid_out.ds.attrs`, despite attributes generated by the regridding process. Not recommended.\n", "\n", - "\n", "### In the following an example showing the function application and the effect of the adaptive masking." ] }, diff --git a/docs/readme.rst b/docs/readme.rst index 929943b4..72a33558 100644 --- a/docs/readme.rst +++ b/docs/readme.rst @@ -1,5 +1 @@ -=========== -Quick Guide -=========== - .. include:: ../README.rst diff --git a/environment.yml b/environment.yml index 9fa922eb..96d9fd7a 100644 --- a/environment.yml +++ b/environment.yml @@ -2,9 +2,9 @@ name: clisops channels: - conda-forge dependencies: - - python >=3.11,<3.14 + - python >=3.11,<3.15 - flit >=3.11.0,<4.0 - - pip >=25.0 + - pip >=25.2 - aiohttp # Needed for HTTPFileSystem - bottleneck >=1.3.1 - cartopy >=0.23.0 @@ -18,7 +18,8 @@ dependencies: - geopandas >=1.0 - jinja2 >=2.11 - loguru >=0.5.3 - - numpy >=1.26.0,<2.3.0 + - numcodecs <0.16.0 # FIXME: Pin required for zarr v2.x compatibility + - numpy >=1.26.0 - packaging >=23.2 - pandas >=2.2.0 - platformdirs >=4.0 @@ -36,30 +37,33 @@ dependencies: # Extras - xesmf >=0.9.2 # # Dev tools and testing - - bump-my-version >=1.2.0 + - bump-my-version >=1.3.0 - coverage >=7.5.0 - - deptry >=0.23.0 - - flake8 >=7.2.0 - - flake8-rst-docstrings >=0.3.0 + - deptry >=0.25.1 + - flake8 >=7.3.0 + - flake8-rst-docstrings >=0.4.0 - h5netcdf >=1.5.0 - - pre-commit >=3.5.0 - - pytest >=8.0.0 - - pytest-cov >=5.0.0 - - pytest-loguru >=0.3.0 - - pytest-xdist >=3.2 - - ruff >=0.12.0 - - tox >=4.24.1 - - watchdog >=4.0.0 + - h5py + - prek >=0.4.0 + - pytest >=9.0.0 + - pytest-cov >=7.0.0 + - pytest-loguru >=0.4.1 + - pytest-timeout >=2.4.0 + - pytest-xdist >=3.2.0 + - ruff >=0.15.9 + - tox >=4.52.0 + - watchdog >=6.0.0 # # Documentation - ipykernel - - ipython >=8.5.0 + - ipython >=8.15.0 - ipython_genutils - jupyter_client - - matplotlib-base >=3.6.0 + - matplotlib-base >=3.8.0 - nbconvert >=7.14.0 - - nbsphinx >=0.9.5 - - sphinx >=7.1.0,<8.2.0 # pinned until nbsphinx supports sphinx 8.2 - - sphinx-autodoc-typehints - - sphinx-codeautolink - - sphinx-copybutton + - nbsphinx >=0.9.8 + - sphinx >=8.2.0 + - sphinx-autobuild >=2024.4.16 + - sphinx-autodoc-typehints >=3.5.0 + - sphinx-codeautolink >=0.16.2 + - sphinx-copybutton >=0.5.2 - sphinx-rtd-theme >=1.0 diff --git a/pyproject.toml b/pyproject.toml index 18c7fa2c..5f2c6c74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,13 +17,13 @@ maintainers = [ readme = {file = "README.rst", content-type = "text/x-rst"} requires-python = ">=3.11.0" keywords = ["clisops", "xarray", "climate", "gis", "subsetting", "operations"] -license = {file = "LICENSE"} +license = "BSD-3-Clause" +license-files = ["LICENSE"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python", @@ -55,7 +55,8 @@ dependencies = [ "geopandas >=1.0", "jinja2 >=2.11", "loguru >=0.5.3", - "numpy >=1.26.0,<2.3.0", + "numcodecs <0.16.0", # FIXME: Pin required for zarr v2.x compatibility + "numpy >=1.26.0", "packaging >=23.2", "pandas >=2.2.0", "platformdirs >=4.0", @@ -74,34 +75,37 @@ dependencies = [ [project.optional-dependencies] dev = [ # Dev tools and testing - "bump-my-version >=1.2.0", + "bump-my-version >=1.3.0", "coverage[toml] >=7.5.0", - "deptry >=0.23.1", + "deptry >=0.25.1", "flake8 >=7.3.0", "flake8-rst-docstrings >=0.4.0", "h5netcdf >=1.5.0", + "h5py", "pip >=25.2", - "pre-commit >=3.5.0", - "pytest >=8.0.0", - "pytest-cov >=5.0.0", - "pytest-loguru >=0.3.0", + "prek >=0.4.0", + "pytest >=9.0.0", + "pytest-cov >=7.0.0", + "pytest-loguru >=0.4.1", + "pytest-timeout >=2.4.0", "pytest-xdist[psutil] >=3.2", - "ruff >=0.14.3", - "tox >=4.30.3", - "watchdog >=4.0.0" + "ruff >=0.15.9", + "tox >=4.52.0", + "watchdog >=6.0.0" ] docs = [ "ipykernel", - "ipython >=8.5.0", + "ipython >=8.15.0", "ipython_genutils", "jupyter_client", - "matplotlib >=3.6.0", + "matplotlib >=3.8.0", "nbconvert >=7.14.0", - "nbsphinx >=0.9.5", - "sphinx >=7.1.0,<8.2", # pinned until nbsphinx supports sphinx 8.2 - "sphinx-autodoc-typehints", - "sphinx-codeautolink", - "sphinx-copybutton", + "nbsphinx >=0.9.8", + "sphinx >=8.2", + "sphinx-autobuild >=2024.4.16", + "sphinx-autodoc-typehints >=3.5.0", + "sphinx-codeautolink >=0.16.2", + "sphinx-copybutton >=0.5.2", "sphinx-rtd-theme >=1.0" ] extras = [ @@ -136,7 +140,7 @@ search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" [tool.codespell] -ignore-words-list = "scrip,nam,te,mye,bu,lik,tread,fo" +ignore-words-list = "bu,fo,lik,mye,nam,scrip,te,tread" [tool.coverage.paths] source = ["clisops", "*/site-packages/clisops/"] @@ -147,13 +151,13 @@ omit = ["*/tests/*.py"] source = ["clisops"] [tool.deptry] +known_first_party = ["clisops"] extend_exclude = ["docs"] ignore_notebooks = true pep621_dev_dependency_groups = ["dev", "docs", "extras"] [tool.deptry.per_rule_ignores] -DEP002 = ["aiohttp", "bottleneck", "cartopy", "fastparquet", "netcdf4", "requests", "s3fs", "zarr"] -DEP003 = ["clisops"] # should be removed when restructuring to src layout +DEP002 = ["aiohttp", "bottleneck", "cartopy", "fastparquet", "netcdf4", "numcodecs", "requests", "s3fs", "zarr"] DEP004 = ["xesmf"] [tool.flit.sdist] @@ -220,8 +224,8 @@ override_SS05 = [ '^Statistics ' ] -[tool.pytest.ini_options] -minversion = "7.0" +[tool.pytest] +minversion = "9.0" addopts = [ "-ra", "--verbose", @@ -229,8 +233,8 @@ addopts = [ "--numprocesses=0", "--maxprocesses=8", "--dist=worksteal", - "--strict-config", - "--strict-markers" + "--strict", + "--timeout=300" ] filterwarnings = ["ignore::UserWarning"] markers = [ diff --git a/tests/data/meridian.json b/tests/data/meridian.json index 6b638c68..995dff56 100644 --- a/tests/data/meridian.json +++ b/tests/data/meridian.json @@ -9,7 +9,7 @@ "coordinates": [ [ [ - 5.8671874999999996, + 5.8671875, 57.326521225217064 ], [ @@ -33,7 +33,7 @@ 52.482780222078226 ], [ - 5.8671874999999996, + 5.8671875, 57.326521225217064 ] ] diff --git a/tests/data/meridian_multi.json b/tests/data/meridian_multi.json index afaa1887..fbefd70b 100644 --- a/tests/data/meridian_multi.json +++ b/tests/data/meridian_multi.json @@ -1,67 +1,67 @@ { - "type": "FeatureCollection", - "features": [ - { - "id": "0", - "type": "Feature", - "properties": {}, - "geometry": { - "type": "MultiPolygon", - "coordinates": [ - [ - [ - [ - -22.148437499999996, - 54.16243396806779 - ], - [ - -29.003906249999996, - 53.85252660044951 - ], - [ - -29.70703125, - 48.22467264956519 - ], - [ - -27.773437499999996, - 42.8115217450979 - ], - [ - -16.34765625, - 47.39834920035926 - ], - [ - -22.148437499999996, - 54.16243396806779 - ] - ] - ], - [ - [ - [ - -23.73046875, - 58.44773280389084 - ], - [ - 10.1953125, - 58.26328705248601 - ], - [ - 8.0859375, - 65.29346780107583 - ], - [ - -21.26953125, - 64.77412531292873 - ], - [ - -23.73046875, - 58.44773280389084 - ] - ] - ] - ] - } - } - ] + "type": "FeatureCollection", + "features": [ + { + "id": "0", + "type": "Feature", + "properties": {}, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -22.148437499999996, + 54.16243396806779 + ], + [ + -29.003906249999996, + 53.85252660044951 + ], + [ + -29.70703125, + 48.22467264956519 + ], + [ + -27.773437499999996, + 42.8115217450979 + ], + [ + -16.34765625, + 47.39834920035926 + ], + [ + -22.148437499999996, + 54.16243396806779 + ] + ] + ], + [ + [ + [ + -23.73046875, + 58.44773280389084 + ], + [ + 10.1953125, + 58.26328705248601 + ], + [ + 8.0859375, + 65.29346780107583 + ], + [ + -21.26953125, + 64.77412531292873 + ], + [ + -23.73046875, + 58.44773280389084 + ] + ] + ] + ] + } + } + ] } diff --git a/tests/data/multi_regions.json b/tests/data/multi_regions.json index 8c0d9a7a..f5d797b1 100644 --- a/tests/data/multi_regions.json +++ b/tests/data/multi_regions.json @@ -3,7 +3,9 @@ "features": [ { "type": "Feature", - "properties": {"id": "Québec"}, + "properties": { + "id": "Québec" + }, "geometry": { "type": "Polygon", "coordinates": [ @@ -46,7 +48,9 @@ }, { "type": "Feature", - "properties": {"id": "Europe"}, + "properties": { + "id": "Europe" + }, "geometry": { "type": "Polygon", "coordinates": [ @@ -121,7 +125,9 @@ }, { "type": "Feature", - "properties": {"id": "Newfoundland"}, + "properties": { + "id": "Newfoundland" + }, "geometry": { "type": "Polygon", "coordinates": [ diff --git a/tests/test_project_utils.py b/tests/test_project_utils.py index 91baffdb..13369f0b 100644 --- a/tests/test_project_utils.py +++ b/tests/test_project_utils.py @@ -3,7 +3,7 @@ import pytest import xarray as xr -from clisops import config, project_utils +from clisops import project_utils @pytest.fixture(scope="module") @@ -79,12 +79,6 @@ def test_get_project_name(self, mini_esgf_data): project = project_utils.get_project_name(dset) assert project in ["c3s-ipcc-ar6-atlas", "c3s-ipcc-atlas"] - @pytest.mark.xfail(reason="outdated") - def test_get_project_name_badc(self): - dset = "/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/rcp85/mon/atmos/Amon/r1i1p1/latest/tas/*.nc" - project = project_utils.get_project_name(dset) - assert project == "cmip5" - def test_get_project_base_dir(self): cmip5_base_dir = project_utils.get_project_base_dir("cmip5") assert cmip5_base_dir == "/mnt/lustre/work/kd0956/CMIP5/data/cmip5" @@ -128,41 +122,6 @@ def test_files(self): "/siconc_SImon_CESM2_historical_r1i1p1f1_gn_185001-201412.nc" ] - @pytest.mark.xfail(reason="outdated") - def test_fixed_path_mappings(self, write_roocs_cfg, monkeypatch): - # reload the roocs_config - monkeypatch.setenv("ROOCS_CONFIG", write_roocs_cfg) - project_utils.CONFIG = config.reload_config() - - dsm = project_utils.DatasetMapper("proj_test.my.first.test") - assert dsm._data_path == "/projects/test/proj/first/test/something.nc" - assert dsm.files == [] # because these do not exist when globbed - - dsm = project_utils.DatasetMapper("proj_test.my.second.test") - assert dsm._data_path == "/projects/test/proj/second/test/data_*.txt" - assert dsm.files == [] # because these do not exist when globbed - - dsm = project_utils.DatasetMapper("proj_test.my.unknown") - assert dsm._data_path == "/projects/test/proj/my/unknown" - - # reset the config - monkeypatch.delenv("ROOCS_CONFIG") - project_utils.CONFIG = config.reload_config() - - @pytest.mark.xfail(reason="outdated") - def test_fixed_path_modifiers(self, write_roocs_cfg, monkeypatch): - """Tests how modifiers can change the fixed path mappings.""" - # reload the roocs_config - monkeypatch.setenv("ROOCS_CONFIG", write_roocs_cfg) - project_utils.CONFIG = config.reload_config() - - dsm = project_utils.DatasetMapper("proj_test.another.sun.test") - assert dsm._data_path == "/projects/test/proj/good/test/sun.nc" - - # reset the config - monkeypatch.delenv("ROOCS_CONFIG") - project_utils.CONFIG = config.reload_config() - @pytest.mark.skipif(os.path.isdir("/badc") is False, reason="data not available") def test_get_filepaths(): @@ -203,70 +162,6 @@ def test_get_filepaths(): ) -class TestDset: - @pytest.mark.xfail(reason="outdated") - def test_derive_dset(self): - from clisops.project_utils import derive_dset - - # c3s-cmip6 - dset = "c3s-cmip6.CMIP.MIROC.MIROC6.historical.r1i1p1f1.SImon.siconc.gn.latest" - ds_id = derive_dset(dset) - - assert ds_id == "/badc/cmip6/data/CMIP6/CMIP/MIROC/MIROC6/historical/r1i1p1f1/SImon/siconc/gn/latest" - - # cmip5 - dset = "cmip5.output1.ICHEC.EC-EARTH.historical.day.atmos.day.r1i1p1.tas.v20131231" - ds_id = derive_dset(dset) - - assert ds_id == "/badc/cmip5/data/cmip5/output1/ICHEC/EC-EARTH/historical/day/atmos/day/r1i1p1/tas/v20131231" - - # c3s-cmip6-decadal - dset = "c3s-cmip6-decadal.DCPP.MOHC.HadGEM3-GC31-MM.dcppA-hindcast.s1995-r1i1p1f2.Amon.tas.gn.v20200417" - ds_id = derive_dset(dset) - - assert ( - ds_id - == "/badc/cmip6/data/CMIP6/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/s1995-r1i1p1f2/Amon/tas/gn/v20200417" - ) - - # c3s-cica-atlas - dset = "c3s-cica-atlas.cd.CMIP6.historical.yr" - ds_id = derive_dset(dset) - - assert ds_id == "/pool/data/c3s-cica-atlas/cd/CMIP6/historical/yr" - - # c3s-ipcc-ar6-atlas - dset = "c3s-ipcc-ar6-atlas.cd.CMIP6.historical.yr" - ds_id = derive_dset(dset) - - assert ds_id == "/pool/data/c3s-ipcc-ar6-atlas/cd/CMIP6/historical/yr" - - @pytest.mark.xfail(reason="outdated") - def test_switch_dset(self): - from clisops.project_utils import switch_dset - - dset = "/badc/cmip6/data/CMIP6/CMIP/MIROC/MIROC6/historical/r1i1p1f1/SImon/siconc/gn/latest/*.nc" - ds_id = switch_dset(dset) - - assert ds_id == "c3s-cmip6.CMIP.MIROC.MIROC6.historical.r1i1p1f1.SImon.siconc.gn.latest" - - @pytest.mark.xfail(reason="outdated") - def test_switch_dset_modified_config(self, write_roocs_cfg, monkeypatch): - # reload the roocs_config - monkeypatch.setenv("ROOCS_CONFIG", write_roocs_cfg) - project_utils.CONFIG = config.reload_config() - - dset = "/badc/cmip6/data/CMIP6/CMIP/MIROC/MIROC6/historical/r1i1p1f1/SImon/siconc/gn/latest/*.nc" - ds_id = project_utils.switch_dset(dset) - - # The first match is returned when parsing the projects within the roocs.ini file - assert ds_id == "c3s-cmip6-decadal.CMIP.MIROC.MIROC6.historical.r1i1p1f1.SImon.siconc.gn.latest" - - # reset the config - monkeypatch.delenv("ROOCS_CONFIG") - project_utils.CONFIG = config.reload_config() - - def test_unknown_fpath_force(): dset = "/tmp/tmpxi6d78ng/subset_tttaum9d/rlds_Amon_IPSL-CM6A-LR_historical_r1i1p1f1_gr_19850116-20141216.nc" @@ -324,20 +219,3 @@ def test_filemapper(self): ] assert dm.data_path == "/badc/cmip6/data/CMIP6/CMIP/MIROC/MIROC6/amip/r1i1p1f1/day/tas/gn/latest" assert dm.ds_id == "c3s-cmip6.CMIP.MIROC.MIROC6.amip.r1i1p1f1.day.tas.gn.latest" - - -@pytest.mark.xfail(reason="outdated") -def test_url_to_file_path(cds_domain): - from clisops.project_utils import url_to_file_path - - url = ( - f"{cds_domain}/thredds/fileServer/esg_c3s-cmip6/CMIP/E3SM-Project/E3SM-1-1" - "/historical/r1i1p1f1/Amon/rlus/gr/v20191211/rlus_Amon_E3SM-1-1_historical_r1i1p1f1_gr_200001-200912.nc" - ) - fpath = url_to_file_path(url) - - assert ( - fpath == "/badc/cmip6/data/CMIP6/CMIP/E3SM-Project/E3SM-1-1" - "/historical/r1i1p1f1/Amon/rlus/gr/v20191211" - "/rlus_Amon_E3SM-1-1_historical_r1i1p1f1_gr_200001-200912.nc" - ) diff --git a/tox.ini b/tox.ini index 1dfff6ec..083506ba 100644 --- a/tox.ini +++ b/tox.ini @@ -1,32 +1,30 @@ [tox] -min_version = 4.30.3 envlist = - py{3.11,3.12,3.13} + py3.{11,12,13,14} lint docs -requires = - flit >=3.11.0,<4.0 - pip >=25.2 opts = -v [gh] python = - 3.11 = py3.11-coverage - 3.12 = py3.12-netcdf4-coverage + 3.11 = py3.11 + 3.12 = py3.12-netcdf4 3.13 = py3.13 + 3.14 = py3.14 [testenv:lint] skip_install = True basepython = python deps = - deptry >=0.23.1 + deptry >=0.25.1 flake8 >=7.3.0 flake8-rst-docstrings >=0.4.0 - ruff >=0.14.3 + ruff >=0.15.9 commands_pre = pip list commands = make lint +commands_post = allowlist_externals = make @@ -37,12 +35,13 @@ extras = docs deps = commands = make docs +commands_post = allowlist_externals = make [testenv] setenv = - PYTEST_ADDOPTS = -m "not slow" --numprocesses=logical --durations=10 --cov=clisops --cov-report=term-missing + PYTEST_ADDOPTS = -m "not slow" --numprocesses=logical --durations=10 --cov=clisops --cov-report=lcov PYTHONPATH = {toxinidir} passenv = CI @@ -53,15 +52,12 @@ extras = dev install_command = python -m pip install --no-user {opts} {packages} download = True deps = - coverage: coveralls >=4.0.1 netcdf4: netcdf4 >=1.6.3 upstream: -r CI/requirements_upstream.txt commands_pre = python -m pip list python -m pip check commands = - ; Recompile h5py to avoid issues with different numpy versions - python -m pip install --no-user --upgrade --no-deps --force-reinstall --no-binary h5py h5py>=3.12.1 pytest {posargs} -commands_post: - coverage: - coveralls +commands_post = + coverage report