From f17647f7cde5cbc2a0fdb3f17f6a1a2f9d545a35 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 21 May 2026 14:40:38 -0400 Subject: [PATCH 01/15] Adapt finch for xclim 0.61 and xsdba 0.6 --- environment-dev.yml | 2 +- environment-docs.yml | 8 ++++---- environment.yml | 14 +++++++------- pyproject.toml | 16 +++++++++------- src/finch/cli.py | 4 ++++ src/finch/processes/__init__.py | 3 ++- src/finch/processes/ensemble_utils.py | 8 ++++++-- src/finch/processes/utils.py | 9 +++------ src/finch/processes/wps_base.py | 15 ++++++++++----- src/finch/processes/wps_hourly_to_daily.py | 11 ++++------- src/finch/processes/wps_sdba.py | 13 ++++++------- src/finch/processes/wpsio.py | 2 +- tests/conftest.py | 2 +- tests/test_wps_caps.py | 2 +- tests/test_wps_sdba.py | 7 +++---- tests/test_wps_xclim_indices.py | 13 +++++++------ tests/test_wps_xsubset_point.py | 14 +++++--------- 17 files changed, 74 insertions(+), 69 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 7c182c5f..6410e42b 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: finch channels: - conda-forge dependencies: - - python >=3.10,<3.13 + - python >=3.10,<3.14 - pywps >=4.6 - jinja2 >=3.1.4 - click >=8.1.7 diff --git a/environment-docs.yml b/environment-docs.yml index bb9a784b..9d27730a 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -2,7 +2,7 @@ name: finch channels: - conda-forge dependencies: - - python >=3.10,<3.13 + - python >=3.10,<3.14 - anyascii >=0.3.0 - birdy >=0.8.1 - ipython >=8.5.0 @@ -11,7 +11,7 @@ dependencies: - pandas >=2.2.0 - pywps >=4.5.1 - setuptools >=71.0.0 - - sphinx >=7.0.0,<8.2.0 # Pinned until nbsphinx supports Sphinx 8.2 + - sphinx >=7.0.0 - sphinxcontrib-bibtex >=2.6.0 - - xarray >=2023.11.0,<2025.3.0 - - xclim =0.52.2 # remember to match xclim version in requirements_docs.txt as well + - xarray >=2023.11.0 + - xclim =0.61.0 # remember to match xclim version in requirements_docs.txt as well diff --git a/environment.yml b/environment.yml index 695afcc5..64bd4775 100644 --- a/environment.yml +++ b/environment.yml @@ -2,19 +2,19 @@ name: finch channels: - conda-forge dependencies: - - python >=3.10,<3.13 + - python >=3.10,<3.14 - pip >=24.2.0 - anyascii >=0.3.0 - cftime >=1.4.1 - cf_xarray >=0.9.3 - click >=8.0.0 - clisops >=0.16.2 - - dask >=2023.5.1,<2025.3.0 + - dask >=2023.5.1 - distributed - geopandas >=1.0 - jinja2 >=3.1.4 - - netcdf4 <=1.7.2 - - numcodecs <0.16.0 + - netcdf4 + - numcodecs - numpy >=1.25.0 - pandas >=2.2.0 - parse @@ -27,7 +27,7 @@ dependencies: - setuptools >=71.0.0 - siphon - werkzeug >=3.0.6 - - xarray >=2023.11.0,<2025.03.0 - - xclim =0.52.2 # remember to match xclim version in requirements_docs.txt as well + - xarray >=2023.11.0 + - xclim =0.61.0 # remember to match xclim version in requirements_docs.txt as well - xesmf >=0.8.2,!=0.8.8 - - xscen =0.10.0 # remember to match xscen version in environment.yml as well + - xscen =0.15.0 # remember to match xscen version in environment.yml as well diff --git a/pyproject.toml b/pyproject.toml index bc91e5d8..e9967fae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering :: Atmospheric Science" ] dynamic = ["description", "version"] @@ -37,11 +39,11 @@ dependencies = [ "cftime >=1.4.1", "click >=8.1.7", "clisops >=0.16.2", - "dask[complete] >=2023.5.1,<2025.4.0", + "dask[complete] >=2023.5.1", "geopandas >=1.0", "jinja2 >=3.1.4", - "netcdf4 <=1.7.2", - "numcodecs <0.16.0", + "netcdf4", + "numcodecs", "numpy >=1.25.0", "pandas >=2.2.0", "parse", @@ -52,10 +54,10 @@ dependencies = [ "scipy >=1.9.0", "sentry-sdk", "siphon", - "xarray >=2023.11.0,<2025.4.0", - "xclim ==0.52.2", # remember to match xclim version in environment.yml as well - "xesmf >=0.6.2,!=0.8.8", - "xscen ==0.10.0", # remember to match xscen version in environment.yml as well + "xarray >=2023.11.0", + "xclim ==0.61.0", # remember to match xclim version in environment.yml as well + "xesmf >=0.8.2,!=0.8.8", + "xscen ==0.15.0", # remember to match xscen version in environment.yml as well "werkzeug>=3.0.6" ] diff --git a/src/finch/cli.py b/src/finch/cli.py index 106581bb..dc5337e5 100644 --- a/src/finch/cli.py +++ b/src/finch/cli.py @@ -246,3 +246,7 @@ def start( else: # no daemon _run(app, bind_host=bind_host) + + +if __name__ == "__main__": + cli() diff --git a/src/finch/processes/__init__.py b/src/finch/processes/__init__.py index 1a722746..09b1d081 100644 --- a/src/finch/processes/__init__.py +++ b/src/finch/processes/__init__.py @@ -1,6 +1,7 @@ # noqa: D104 import logging +import xarray as xr from pywps.configuration import get_config_value from xclim.core.indicator import registry as xclim_registry @@ -71,7 +72,7 @@ def filter_func(elem): def get_processes(): """Get wps processes using the current global `pywps` configuration.""" indicators = get_indicators( - realms=["atmos", "land", "seaIce"], exclude=not_implemented + realms=["convert", "atmos", "land", "seaIce"], exclude=not_implemented ) mod_dict = get_virtual_modules() for mod in mod_dict.keys(): diff --git a/src/finch/processes/ensemble_utils.py b/src/finch/processes/ensemble_utils.py index 86a5e804..5220ded1 100644 --- a/src/finch/processes/ensemble_utils.py +++ b/src/finch/processes/ensemble_utils.py @@ -21,7 +21,7 @@ from xclim import ensembles from xclim.core.calendar import days_since_to_doy, doy_to_days_since, percentile_doy from xclim.core.indicator import Indicator -from xclim.indicators.atmos import tg +from xclim.indicators.convert import mean_temperature_from_max_and_min from xscen.aggregate import climatological_op, compute_deltas, spatial_mean from . import wpsio @@ -52,7 +52,11 @@ def _percentile_doy(var: xr.DataArray, perc: int) -> xr.DataArray: variable_computations = { - "tas": {"inputs": ["tasmin", "tasmax"], "args": [], "function": tg}, + "tas": { + "inputs": ["tasmin", "tasmax"], + "args": [], + "function": mean_temperature_from_max_and_min, + }, "tasmax_per": { "inputs": ["tasmax"], "args": ["perc_tasmax"], diff --git a/src/finch/processes/utils.py b/src/finch/processes/utils.py index 5b66923a..eceed2ef 100644 --- a/src/finch/processes/utils.py +++ b/src/finch/processes/utils.py @@ -262,12 +262,10 @@ def compute_indices( # noqa: D103 ) options = {name: kwds.pop(name) for name in INDICATOR_OPTIONS if name in kwds} - with xclim_options.set_options(**options): - out = func(**kwds) + with xclim_options.set_options(as_dataset=True, **options): + output_dataset = func(**kwds) - output_dataset = xr.Dataset( - data_vars=None, coords=out.coords, attrs=global_attributes - ) + output_dataset.attrs.update(global_attributes) # fix frequency of computed output (xclim should handle this) if output_dataset.attrs.get("frequency") == "day" and "freq" in kwds: @@ -280,7 +278,6 @@ def compute_indices( # noqa: D103 } output_dataset.attrs["frequency"] = conversions.get(kwds["freq"], "day") - output_dataset[out.name] = out return output_dataset diff --git a/src/finch/processes/wps_base.py b/src/finch/processes/wps_base.py index c637b7b7..f0dbd5d5 100644 --- a/src/finch/processes/wps_base.py +++ b/src/finch/processes/wps_base.py @@ -162,7 +162,7 @@ def convert_xclim_inputs_to_pywps( InputKind.NUMBER_SEQUENCE: "integer", InputKind.STRING: "string", InputKind.DAY_OF_YEAR: "string", - InputKind.DATE: "datetime", + InputKind.DATE: "dateTime", } if parse_percentiles and parent is None: @@ -232,7 +232,10 @@ def convert_xclim_inputs_to_pywps( def make_freq( # noqa: D103 - name, default="YS", abstract="", allowed=("YS", "MS", "QS-DEC", "YS-JAN", "YS-JUL") + name, + default="YS", + abstract="", + allowed=("YS", "MS", "QS-DEC", "YS-JAN", "YS-JUL", "YS-OCT"), ): try: return LiteralInput( @@ -245,12 +248,14 @@ def make_freq( # noqa: D103 default=default, allowed_values=allowed, ) - except pywps.exceptions.InvalidParameterValue: - print(name, default, abstract, allowed) + except pywps.exceptions.InvalidParameterValue as err: + raise NotImplementedError( + f"Finch can't parse frequency argument: {name=}, {default=}, {abstract=}, {allowed=}" + ) from err def make_nc_input(name): # noqa: D103 - desc = xclim.core.utils.VARIABLES.get(name, {}).get("description", "") + desc = xclim.core.VARIABLES.get(name, {}).get("description", "") return ComplexInput( name, "Resource", diff --git a/src/finch/processes/wps_hourly_to_daily.py b/src/finch/processes/wps_hourly_to_daily.py index 18753e4a..b96f2573 100644 --- a/src/finch/processes/wps_hourly_to_daily.py +++ b/src/finch/processes/wps_hourly_to_daily.py @@ -109,10 +109,8 @@ def _hourly_to_daily( ) -> xr.Dataset: """Convert an hourly time series to a daily time series.""" # Validate missing values algorithm options - kls = MISSING_METHODS[check_missing] - missing = kls.execute - if missing_options: - kls.validate(**missing_options) + if check_missing != "skip": + missing = MISSING_METHODS[check_missing](**(missing_options or {})) # Resample to daily out = getattr(ds.resample(time="D"), reducer)(keep_attrs=True) @@ -131,10 +129,9 @@ def _hourly_to_daily( # Compute missing values mask if check_missing != "skip": + srcfreq = xr.infer_freq(ds.time) or "h" for key, da in ds.data_vars.items(): - mask = missing( - da, freq="D", src_timestep="H", options=missing_options, indexer={} - ) + mask = missing(da, freq="D", src_timestep=srcfreq) out[key] = out[key].where(~mask) return out diff --git a/src/finch/processes/wps_sdba.py b/src/finch/processes/wps_sdba.py index 5c1a0a63..e7661dfe 100644 --- a/src/finch/processes/wps_sdba.py +++ b/src/finch/processes/wps_sdba.py @@ -3,17 +3,16 @@ Statistical downscaling and bias adjustment =========================================== -Expose xclim.sdba algorithms as WPS. +Expose xsdba algorithms as WPS. For the moment, both train-adjust operations are bundled into a single process. """ import logging from pathlib import Path -import xclim +import xsdba from pywps import FORMATS, ComplexInput, ComplexOutput, LiteralInput -from xclim.core.calendar import convert_calendar -from xclim.sdba.utils import ADDITIVE, MULTIPLICATIVE +from xsdba.utils import ADDITIVE, MULTIPLICATIVE from . import wpsio from .utils import ( @@ -167,7 +166,7 @@ def _log(message, percentage): name = variable or list(ds.data_vars)[0] # Force calendar to noleap and rechunk - res[key] = convert_calendar(ds[name], "noleap").chunk({"time": -1}) + res[key] = ds[name].convert_calendar("noleap").chunk({"time": -1}) elif key in group_args: group[key] = single_input_or_none(request.inputs, key) @@ -180,10 +179,10 @@ def _log(message, percentage): _log("Successfully read inputs from request.", 1) - group = xclim.sdba.Grouper(**group) + group = xsdba.Grouper(**group) _log("Grouper object created.", 2) - bc = xclim.sdba.EmpiricalQuantileMapping.train( + bc = xsdba.EmpiricalQuantileMapping.train( res["ref"], res["hist"], **train, group=group ) diff --git a/src/finch/processes/wpsio.py b/src/finch/processes/wpsio.py index 62cf1f96..cd18a519 100644 --- a/src/finch/processes/wpsio.py +++ b/src/finch/processes/wpsio.py @@ -256,7 +256,7 @@ def get_ensemble_inputs(novar=False): abstract="Method used to determine which aggregations should be considered missing.", data_type="string", default=OPTIONS[CHECK_MISSING], - allowed_values=list(MISSING_METHODS.keys()), + allowed_values=list(MISSING_METHODS.keys()) + ["skip"], min_occurs=0, ) diff --git a/tests/conftest.py b/tests/conftest.py index 1756860d..6c0a73e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -239,6 +239,6 @@ def hourly_dataset(tmp_path_factory): # noqa: F811 a[0] = np.nan return _write_dataset( "pr_hr", - timeseries(values=a, variable="pr", freq="H"), + timeseries(values=a, variable="pr", freq="h"), tmp_path_factory.mktemp("hourly_ds"), ) diff --git a/tests/test_wps_caps.py b/tests/test_wps_caps.py index 006df0d8..599e4564 100644 --- a/tests/test_wps_caps.py +++ b/tests/test_wps_caps.py @@ -45,7 +45,7 @@ def mock_config_get(*args, **kwargs): ).split() indicators = get_indicators( - realms=["atmos", "land", "seaIce"], exclude=not_implemented + realms=["atmos", "convert", "land", "seaIce"], exclude=not_implemented ) mod_dict = get_virtual_modules() for mod in mod_dict.keys(): diff --git a/tests/test_wps_sdba.py b/tests/test_wps_sdba.py index 81d8b848..c2820441 100644 --- a/tests/test_wps_sdba.py +++ b/tests/test_wps_sdba.py @@ -5,8 +5,7 @@ import xarray as xr from pywps import Service from pywps.tests import assert_response_success, client_for -from xclim.core.calendar import convert_calendar -from xclim.sdba.utils import ADDITIVE, MULTIPLICATIVE +from xsdba.utils import ADDITIVE, MULTIPLICATIVE from _common import CFG_FILE, get_output from finch.processes import EmpiricalQuantileMappingProcess @@ -38,9 +37,9 @@ def test_wps_empirical_quantile_mapping(netcdf_sdba_ds, kind, name): out = get_output(resp.xml) p = xr.open_dataset(out["output"][7:])[name] - uc = convert_calendar(u, "noleap") + uc = u.convert_calendar("noleap") middle = ((uc > 1e-2) * (uc < 0.99)).data ref = xr.open_dataset(sdba_ds[f"qdm_{name}_ref"])[name] - refc = convert_calendar(ref, "noleap") + refc = ref.convert_calendar("noleap") np.testing.assert_allclose(p[middle], refc[middle], rtol=0.03) diff --git a/tests/test_wps_xclim_indices.py b/tests/test_wps_xclim_indices.py index 1574ced1..4b559272 100644 --- a/tests/test_wps_xclim_indices.py +++ b/tests/test_wps_xclim_indices.py @@ -31,15 +31,19 @@ def _get_output_standard_name(process_identifier): @pytest.mark.parametrize( "indicator", - get_indicators(realms=["atmos", "land", "seaIce"], exclude=not_implemented), + get_indicators( + realms=["convert", "atmos", "land", "seaIce"], exclude=not_implemented + ), ) def test_indicators_processes_discovery(indicator): process = make_xclim_indicator_process(indicator, "Process", XclimIndicatorBase) assert indicator.identifier == process.identifier # Remove args not supported by finch: we remove special kinds, # 50 is "kwargs". 70 is Dataset ('ds') and 99 is "unknown". All normal types are 0-9. + # 10 is dict unsupported by pyWPS + # 11 is "mask" which is bool | float | DataArray, not yet supported in finch parameters = { - k for k, v in indicator.parameters.items() if v.kind < 50 or k == "indexer" + k for k, v in indicator.parameters.items() if v.kind < 10 or k == "indexer" } parameters.add("check_missing") parameters.add("missing_options") @@ -172,7 +176,6 @@ def test_heat_wave_frequency_window_thresh_parameters(client, netcdf_datasets): ds = xr.open_dataset(outputs[0]) assert ds.attrs["frequency"] == "yr" - assert ds.heat_wave_frequency.standard_name == _get_output_standard_name(identifier) def test_heat_wave_index_thresh_parameter(client, netcdf_datasets): @@ -184,8 +187,6 @@ def test_heat_wave_index_thresh_parameter(client, netcdf_datasets): outputs = execute_process(client, identifier, inputs) ds = xr.open_dataset(outputs[0]) - assert ds["heat_wave_index"].standard_name == _get_output_standard_name(identifier) - def test_missing_options(client, netcdf_datasets): identifier = "tg_mean" @@ -311,7 +312,7 @@ def test_two_nondefault_variable_name(client, tmp_path): def test_degree_days_exceedance_date(client, tmp_path): identifier = "degree_days_exceedance_date" - tas = open_dataset("FWI/GFWED_sample_2017.nc", branch="v2023.12.14").tas + tas = open_dataset("FWI/GFWED_sample_2017.nc").tas tas.attrs.update( cell_methods="time: mean within days", standard_name="air_temperature" ) diff --git a/tests/test_wps_xsubset_point.py b/tests/test_wps_xsubset_point.py index ceb2f730..bd5d6d9b 100644 --- a/tests/test_wps_xsubset_point.py +++ b/tests/test_wps_xsubset_point.py @@ -65,14 +65,8 @@ def test_thredds(): client = client_for( Service(processes=[SubsetGridPointProcess()], cfgfiles=CFG_FILE) ) - fn1 = ( - "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/" - "birdhouse/disk2/cmip5/MRI/rcp85/fx/atmos/r0i0p0/sftlf/sftlf_fx_MRI-CGCM3_rcp85_r0i0p0.nc" - ) - fn2 = ( - "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/" - "birdhouse/disk2/cmip5/MRI/rcp85/fx/atmos/r0i0p0/orog/orog_fx_MRI-CGCM3_rcp85_r0i0p0.nc" - ) + fn1 = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/xclim/HadGEM2-CC_360day/pr_day_HadGEM2-CC_rcp85_r1i1p1_na10kgrid_qm-moving-50bins-detrend_2095.nc" + fn2 = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/xclim/HadGEM2-CC_360day/tasmax_day_HadGEM2-CC_rcp85_r1i1p1_na10kgrid_qm-moving-50bins-detrend_2095.nc" datainputs = ( f"resource=files@xlink:href={fn1};" @@ -103,7 +97,9 @@ def test_bad_link_on_thredds(): f"?service=WPS&request=Execute&version=1.0.0&identifier=subset_gridpoint&datainputs={datainputs}" ) - assert "NetCDF: file not found" in resp.response[0].decode() + assert ("NetCDF: file not found" in resp.response[0].decode()) or ( + "NetCDF: Unknown file format" in resp.response[0].decode() + ) def test_bad_link_on_fs(): From 5f1744c3a7e4ae7e28f66f0e16bac46940bdb996 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 21 May 2026 15:46:21 -0400 Subject: [PATCH 02/15] Decrease warnings --- src/finch/processes/ensemble_utils.py | 8 +++--- src/finch/processes/subset.py | 9 ++++--- src/finch/processes/utils.py | 18 +++++++------ src/finch/processes/wps_base.py | 28 ++++++++++++-------- tests/test_wps_ensemble.py | 38 +++++++++++++-------------- tests/test_wps_xclim_indices.py | 2 +- tests/test_wps_xsubset_bbox.py | 2 +- tests/test_wps_xsubset_point.py | 2 +- 8 files changed, 59 insertions(+), 48 deletions(-) diff --git a/src/finch/processes/ensemble_utils.py b/src/finch/processes/ensemble_utils.py index 5220ded1..852aabf9 100644 --- a/src/finch/processes/ensemble_utils.py +++ b/src/finch/processes/ensemble_utils.py @@ -364,7 +364,7 @@ def make_ensemble( # noqa: D103 region: dict | None = None, ) -> None: ensemble = ensembles.create_ensemble( - files, realizations=[file.stem for file in files] + files, realizations=[file.stem for file in files], decode_timedelta=False ) # make sure we have data starting in 1950 ensemble = ensemble.sel(time=(ensemble.time.dt.year >= 1950)) @@ -410,7 +410,7 @@ def make_ensemble( # noqa: D103 dslist.extend([compute_deltas(ds=ensemble, reference_horizon=hori)]) if len(dslist) > 1: - ensemble = xr.merge(dslist) + ensemble = xr.merge(dslist, compat="override") if spatavg: # ensemble = ensemble.mean(dim=average_dims) @@ -727,7 +727,9 @@ def ensemble_common_handler( # noqa: C901,D103 ] ensemble = xr.concat( - ensembles, dim=xr.DataArray(scenarios, dims=("scenario",), name="scenario") + ensembles, + dim=xr.DataArray(scenarios, dims=("scenario",), name="scenario"), + join="outer", ) if convert_to_csv: diff --git a/src/finch/processes/subset.py b/src/finch/processes/subset.py index 33b0abd4..17c35c52 100644 --- a/src/finch/processes/subset.py +++ b/src/finch/processes/subset.py @@ -108,7 +108,7 @@ def _subset(resource: ComplexInput): else: subsetted = subsetted.expand_dims("region") - if not all(subsetted.dims.values()): + if not all(subsetted.sizes.values()): msg = f"Subset is empty for dataset: {resource.url}" LOGGER.warning(msg) return @@ -185,7 +185,7 @@ def _subset(resource): except ValueError: subsetted = False - if subsetted is False or not all(subsetted.dims.values()): + if subsetted is False or not all(subsetted.sizes.values()): msg = f"Subset is empty for dataset: {resource.url}" LOGGER.warning(msg) return @@ -251,6 +251,7 @@ def finch_average_shape( shape = gpd.read_file(shp) if tolerance > 0: shape["geometry"] = shape.simplify(tolerance) + shape["geometry"] = shape.geometry.segmentize(1) n_files = len(netcdf_inputs) count = 0 @@ -277,7 +278,7 @@ def finch_average_shape( dataset = subset_time(dataset, start_date=start_date, end_date=end_date) averaged = average_shape(dataset, shape) - if not all(averaged.dims.values()): + if not all(averaged.sizes.values()): msg = f"Average is empty for dataset: {resource.url}" LOGGER.warning(msg) return @@ -343,7 +344,7 @@ def _subset(resource): end_date=end_date, ) - if not all(subsetted.dims.values()): + if not all(subsetted.sizes.values()): msg = f"Subset is empty for dataset: {resource.url}" LOGGER.warning(msg) return diff --git a/src/finch/processes/utils.py b/src/finch/processes/utils.py index eceed2ef..51746981 100644 --- a/src/finch/processes/utils.py +++ b/src/finch/processes/utils.py @@ -61,17 +61,19 @@ def get_virtual_modules(): """Load virtual modules.""" modules = {} if modfiles := get_config_value("finch", "xclim_modules"): - for modfile in modfiles.split(","): - if Path(modfile).is_absolute(): - mod = build_indicator_module_from_yaml(Path(modfile)) - else: - mod = build_indicator_module_from_yaml( - Path(__file__).parent.parent.joinpath(modfile) - ) + for modfile in map(Path, modfiles.split(",")): + mod = getattr(xclim.indicators, modfile.stem, None) + if mod is None: + if modfile.is_absolute(): + mod = build_indicator_module_from_yaml(modfile) + else: + mod = build_indicator_module_from_yaml( + Path(__file__).parent.parent / modfile + ) indicators = [] for indname, ind in mod.iter_indicators(): indicators.append(ind.get_instance()) - modules[Path(modfile).name] = dict(indicators=indicators) + modules[modfile.stem] = dict(indicators=indicators) return modules diff --git a/src/finch/processes/wps_base.py b/src/finch/processes/wps_base.py index f0dbd5d5..06ef7e1e 100644 --- a/src/finch/processes/wps_base.py +++ b/src/finch/processes/wps_base.py @@ -10,7 +10,7 @@ from pywps import FORMATS, ComplexInput, LiteralInput, Process from pywps.app.Common import Metadata from pywps.app.exceptions import ProcessError -from sentry_sdk import configure_scope +from sentry_sdk import get_current_scope from xclim.core.utils import InputKind LOGGER = logging.getLogger("PYWPS") @@ -70,15 +70,15 @@ def sentry_configure_scope(self, request): When sentry is not initialized, this won't add any overhead. """ - with configure_scope() as scope: - scope.set_extra("identifier", self.identifier) - scope.set_extra("request_uuid", str(self.uuid)) - if request.http_request: - # if the request has been put in the `stored_requests` table by pywps - # the original request.http_request is not available anymore - scope.set_extra("host", request.http_request.host) - scope.set_extra("remote_addr", request.http_request.remote_addr) - scope.set_extra("xml_request", request.http_request.data) + scope = get_current_scope() + scope.set_extra("identifier", self.identifier) + scope.set_extra("request_uuid", str(self.uuid)) + if request.http_request: + # if the request has been put in the `stored_requests` table by pywps + # the original request.http_request is not available anymore + scope.set_extra("host", request.http_request.host) + scope.set_extra("remote_addr", request.http_request.remote_addr) + scope.set_extra("xml_request", request.http_request.data) class FinchProgressBar(ProgressBar): @@ -133,7 +133,13 @@ def make_xclim_indicator_process( process = process_class() process.translations = { locale: xclim.core.locales.get_local_attrs( - xci.identifier.upper(), locale, append_locale_name=False + ( + xci.identifier.upper() + if xci._registry_id.startswith("xclim") + else xci._registry_id + ), + locale, + append_locale_name=False, ) for locale in xclim.core.locales.list_locales() } diff --git a/tests/test_wps_ensemble.py b/tests/test_wps_ensemble.py index 07b33e8f..7b97892a 100644 --- a/tests/test_wps_ensemble.py +++ b/tests/test_wps_ensemble.py @@ -52,8 +52,8 @@ def test_ensemble_hxmax_days_above_grid_point(client): # --- then --- assert len(outputs) == 1 assert Path(outputs[0]).stem.startswith("testens_45_500_73_000_ssp245_ssp585") - ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + ds = open_dataset(outputs[0], decode_timedelta=False) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 12, # there are roughly 4 months in the test datasets @@ -93,7 +93,7 @@ def test_ensemble_spatial_avg_grid_point(client): assert len(outputs) == 1 # assert Path(outputs[0]).stem.startswith("testens_45_500_73_000_ssp245_ssp585") ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 2, "time": 4, # there are roughly 4 months in the test datasets @@ -117,7 +117,7 @@ def test_ensemble_spatial_avg_grid_point(client): assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "time": 4, # there are roughly 4 months in the test datasets "scenario": 2, @@ -220,7 +220,7 @@ def test_ensemble_temporal_avg_bbox(client): assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == {"time": 36, "scenario": 2, "realization": 3, "lat": 12, "lon": 12} ensemble_variables = [ @@ -285,7 +285,7 @@ def test_ensemble_spatial_avg_poly(client): assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "time": 4, # there are roughly 4 months in the test datasets "scenario": 2, @@ -321,7 +321,7 @@ def test_ensemble_spatial_avg_poly_noperc(client): assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) exp_dims = { "realization": 2, "time": 4, # there are roughly 4 months in the test datasets @@ -361,7 +361,7 @@ def test_ensemble_heatwave_frequency_grid_point(client): assert len(outputs) == 1 assert Path(outputs[0]).stem.startswith("testens_46_000_72_800_rcp45") ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 4, # there are roughly 4 months in the test datasets @@ -434,7 +434,7 @@ def test_ensemble_heatwave_frequency_grid_point_no_perc(client): assert len(outputs) == 1 assert Path(outputs[0]).stem.startswith("testens_46_000_72_800_rcp45") ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 4, # there are roughly 4 months in the test datasets @@ -479,7 +479,7 @@ def test_ensemble_dded_grid_point_multiscenario(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 4, # there are roughly 4 months in the test datasets @@ -518,7 +518,7 @@ def test_ensemble_dded_grid_point_multiscenario_noperc(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 4, # there are roughly 4 months in the test datasets @@ -562,7 +562,7 @@ def test_ensemble_heatwave_frequency_bbox(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "lat": 2, "lon": 2, @@ -583,7 +583,7 @@ def test_ensemble_heatwave_frequency_bbox(client): assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == {"time": 4, "scenario": 1} # Spatial average has been taken. ensemble_variables = {k: v for k, v in ds.data_vars.items()} @@ -681,7 +681,7 @@ def test_ensemble_heatwave_frequency_grid_point_dates(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == {"region": 1, "time": 3, "scenario": 1} ensemble_variables = dict(ds.data_vars) @@ -777,8 +777,8 @@ def test_ensemble_compute_intermediate_cold_spell_duration_index_grid_point(clie # --- then --- assert len(outputs) == 1 - ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + ds = open_dataset(outputs[0], decode_timedelta=False) + dims = dict(ds.sizes) assert dims == {"region": 1, "time": 1, "scenario": 1} ensemble_variables = dict(ds.data_vars) @@ -806,7 +806,7 @@ def test_ensemble_compute_intermediate_growing_degree_days_grid_point(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == {"region": 1, "time": 1, "scenario": 1} ensemble_variables = dict(ds.data_vars) @@ -844,7 +844,7 @@ def test_ensemble_heatwave_frequency_polygon(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "lat": 11, "lon": 11, @@ -869,7 +869,7 @@ def test_ensemble_heatwave_frequency_polygon(client): # --- then --- assert len(outputs) == 1 ds = open_dataset(outputs[0]) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == {"time": 4, "scenario": 1} ensemble_variables = dict(ds.data_vars) diff --git a/tests/test_wps_xclim_indices.py b/tests/test_wps_xclim_indices.py index 4b559272..9b37c59f 100644 --- a/tests/test_wps_xclim_indices.py +++ b/tests/test_wps_xclim_indices.py @@ -185,7 +185,7 @@ def test_heat_wave_index_thresh_parameter(client, netcdf_datasets): wps_literal_input("thresh", "30 degC"), ] outputs = execute_process(client, identifier, inputs) - ds = xr.open_dataset(outputs[0]) + ds = xr.open_dataset(outputs[0], decode_timedelta=False) def test_missing_options(client, netcdf_datasets): diff --git a/tests/test_wps_xsubset_bbox.py b/tests/test_wps_xsubset_bbox.py index 8cd511b9..bc7706db 100644 --- a/tests/test_wps_xsubset_bbox.py +++ b/tests/test_wps_xsubset_bbox.py @@ -68,7 +68,7 @@ def test_wps_subsetbbox_dataset(client, outfmt): with zf.open(data_filenames[0]) as f: ds = xr.open_dataset(f) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "lon": 6, "lat": 6, diff --git a/tests/test_wps_xsubset_point.py b/tests/test_wps_xsubset_point.py index bd5d6d9b..91c0d108 100644 --- a/tests/test_wps_xsubset_point.py +++ b/tests/test_wps_xsubset_point.py @@ -146,7 +146,7 @@ def test_wps_subsetpoint_dataset(client, outfmt): with zf.open(data_filenames[0]) as f: ds = xr.open_dataset(f) - dims = dict(ds.dims) + dims = dict(ds.sizes) assert dims == { "region": 1, "time": 100, From a87fbfb1e34194fccfe772b97ebb39eaf463dc24 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 21 May 2026 17:50:52 -0400 Subject: [PATCH 03/15] suggestions from review --- docs/source/conf.py | 2 ++ environment-docs.yml | 4 ++-- environment.yml | 9 +++++---- pyproject.toml | 6 +++--- src/finch/processes/__init__.py | 1 - src/finch/processes/wpsio.py | 4 +++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9a89bfc9..079841da 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -96,6 +96,8 @@ "statsmodels", "unidecode", "xarray", + "xsdba", + "xscen", "zlib", ] diff --git a/environment-docs.yml b/environment-docs.yml index 9d27730a..853c6377 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -2,7 +2,7 @@ name: finch channels: - conda-forge dependencies: - - python >=3.10,<3.14 + - python >=3.11,<3.14 - anyascii >=0.3.0 - birdy >=0.8.1 - ipython >=8.5.0 @@ -14,4 +14,4 @@ dependencies: - sphinx >=7.0.0 - sphinxcontrib-bibtex >=2.6.0 - xarray >=2023.11.0 - - xclim =0.61.0 # remember to match xclim version in requirements_docs.txt as well + - xclim >=0.61,<0.62 # remember to match xclim version in environment.yml and pyproject.toml as well diff --git a/environment.yml b/environment.yml index 64bd4775..fb2407fb 100644 --- a/environment.yml +++ b/environment.yml @@ -2,8 +2,8 @@ name: finch channels: - conda-forge dependencies: - - python >=3.10,<3.14 - - pip >=24.2.0 + - python >=3.11,<3.14 + - pip >=26.1.0 - anyascii >=0.3.0 - cftime >=1.4.1 - cf_xarray >=0.9.3 @@ -28,6 +28,7 @@ dependencies: - siphon - werkzeug >=3.0.6 - xarray >=2023.11.0 - - xclim =0.61.0 # remember to match xclim version in requirements_docs.txt as well + - xclim >=0.61,<0.62 # remember to match xclim version in environment-docs.yml and pyproject.toml as well - xesmf >=0.8.2,!=0.8.8 - - xscen =0.15.0 # remember to match xscen version in environment.yml as well + - xsdba =0.6.1 # remember to match xsdba version in pyproject.toml as well + - xscen =0.15.0 # remember to match xscen version in pyproject.toml as well diff --git a/pyproject.toml b/pyproject.toml index e9967fae..586cd88b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ maintainers = [ {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"}, ] readme = {file = "README.rst", content-type = "text/x-rst"} -requires-python = ">=3.10.0" +requires-python = ">=3.11.0" keywords = ["wps", "pywps", "birdhouse", "finch"] license = {file = "LICENSE"} classifiers = [ @@ -25,7 +25,6 @@ classifiers = [ "Programming Language :: Python", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -55,8 +54,9 @@ dependencies = [ "sentry-sdk", "siphon", "xarray >=2023.11.0", - "xclim ==0.61.0", # remember to match xclim version in environment.yml as well + "xclim >=0.61,<0.62", # remember to match xclim version in environment.yml and environment-docs.yml as well "xesmf >=0.8.2,!=0.8.8", + "xsdna ==0.6.1", # remember to match xsdba version in environment.yml as well "xscen ==0.15.0", # remember to match xscen version in environment.yml as well "werkzeug>=3.0.6" ] diff --git a/src/finch/processes/__init__.py b/src/finch/processes/__init__.py index 09b1d081..2bbd8e20 100644 --- a/src/finch/processes/__init__.py +++ b/src/finch/processes/__init__.py @@ -1,7 +1,6 @@ # noqa: D104 import logging -import xarray as xr from pywps.configuration import get_config_value from xclim.core.indicator import registry as xclim_registry diff --git a/src/finch/processes/wpsio.py b/src/finch/processes/wpsio.py index cd18a519..5d2acda1 100644 --- a/src/finch/processes/wpsio.py +++ b/src/finch/processes/wpsio.py @@ -250,13 +250,15 @@ def get_ensemble_inputs(novar=False): min_occurs=0, ) +_missing_methods = list(MISSING_METHODS.keys()) +_missing_methods.append("skip") check_missing = LiteralInput( "check_missing", "Missing value handling method", abstract="Method used to determine which aggregations should be considered missing.", data_type="string", default=OPTIONS[CHECK_MISSING], - allowed_values=list(MISSING_METHODS.keys()) + ["skip"], + allowed_values=_missing_methods, min_occurs=0, ) From 8b57d0e51b904c0fd2193d49cac200b03c5b51f5 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 21 May 2026 18:08:28 -0400 Subject: [PATCH 04/15] remove non-working mocking from docs - remove non-docs deps from doc env --- docs/source/conf.py | 54 +------------------------------------------- environment-docs.yml | 6 ----- environment.yml | 2 +- pyproject.toml | 6 ++--- 4 files changed, 5 insertions(+), 63 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 079841da..c4b3dc4b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,53 +53,6 @@ "sphinxcontrib.bibtex", ] -# To avoid having to install these and burst memory limit on ReadTheDocs. -# List of all tested working mock imports from all birds so new birds can -# inherit without having to test which work which do not. -if os.environ.get("READTHEDOCS") == "True": - autodoc_mock_imports = [ - "affine", - "bottleneck", - "cairo", - "cartopy", - "cftime", - "cf_xarray", - "clisops", - "dask", - "fiona", - "gdal", - "geopandas", - "geos", - "geotiff", - "hdf4", - "hdf5", - "matplotlib", - "netCDF4", - "numba", - "numpy", - "ocgis", - "osgeo", - "pandas", - "parse", - "proj", - "pyproj", - "rasterio", - "rasterstats", - "scikit-learn", - "scipy", - "sentry_sdk", - "shapely", - "siphon", - "sklearn", - "slugify", - "spotpy", - "statsmodels", - "unidecode", - "xarray", - "xsdba", - "xscen", - "zlib", - ] # Bibliography stuff, for correct xclim docstring formatting # We need to download the reference file from xclim for the correct version. @@ -126,12 +79,7 @@ "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), } -# Monkeypatch constant because the following are mock imports. -# Only works if numpy is actually installed and at the same time being mocked. -# import numpy -# numpy.pi = 3.1416 - -# We are using mock imports in readthedocs, so probably safer to not run the notebooks +# Probably safer to not run the notebooks nbsphinx_execute = "never" # Add any paths that contain templates here, relative to this directory. diff --git a/environment-docs.yml b/environment-docs.yml index 54c8eb78..b5765d68 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -4,15 +4,9 @@ channels: - nodefaults dependencies: - python >=3.11,<3.14 - - anyascii >=0.3.0 - birdy >=0.8.1 - ipython >=8.5.0 - matplotlib-base >=3.5.0 - nbsphinx >=0.9.8 - - pandas >=2.2.0,<3.0 - - pywps >=4.5.1 - - setuptools >=71.0.0 - sphinx >=7.0.0 - sphinxcontrib-bibtex >=2.6.0 - - xarray >=2023.11.0 - - xclim >=0.61,<0.62 # remember to match xclim version in environment.yml and pyproject.toml as well diff --git a/environment.yml b/environment.yml index a6d8885a..76372dcb 100644 --- a/environment.yml +++ b/environment.yml @@ -29,7 +29,7 @@ dependencies: - siphon >=0.10.0 - werkzeug >=3.0.6 - xarray >=2023.11.0 - - xclim >=0.61,<0.62 # remember to match xclim version in environment-docs.yml and pyproject.toml as well + - xclim >=0.61,<0.62 # remember to match xclim version in pyproject.toml as well - xesmf >=0.8.2,!=0.8.8 - xsdba =0.6.1 # remember to match xsdba version in pyproject.toml as well - xscen =0.15.0 # remember to match xscen version in pyproject.toml as well diff --git a/pyproject.toml b/pyproject.toml index 3a86ab95..cd2ed31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ dependencies = [ "sentry-sdk", "siphon", "xarray >=2023.11.0", - "xclim >=0.61,<0.62", # remember to match xclim version in environment.yml and environment-docs.yml as well + "xclim >=0.61,<0.62", # remember to match xclim version in environment.yml as well "xesmf >=0.8.2,!=0.8.8", - "xsdna ==0.6.1", # remember to match xsdba version in environment.yml as well + "xsdba ==0.6.1", # remember to match xsdba version in environment.yml as well "xscen ==0.15.0", # remember to match xscen version in environment.yml as well "werkzeug>=3.0.6", # Temporary fixes @@ -80,7 +80,7 @@ dev = [ "nbsphinx>=0.9.5", "nbval >=0.10.0", "owslib", - "pip >=25.3", + "pip >=26.1", "pre-commit >=3.5.0", "pylint >=3.3.3", "pytest >=8.0.0", From 811c31e943d1ab0682a25656b53d3650781f6c11 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Thu, 21 May 2026 18:29:47 -0400 Subject: [PATCH 05/15] Add builds for 3.13 and 3.14 - help pip who seems to struggle on the ci --- .github/workflows/main.yml | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d654529e..8c997354 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.13" ] + python-version: [ "3.14" ] steps: - name: Harden Runner uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 @@ -53,7 +53,7 @@ jobs: shell: bash -l {0} strategy: matrix: - python-version: [ "3.10", "3.11", "3.12" ] + python-version: [ "3.11", "3.12", "3.13", "3.14" ] steps: - name: Harden Runner uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 diff --git a/pyproject.toml b/pyproject.toml index cd2ed31d..fb9d37d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dependencies = [ [project.optional-dependencies] dev = [ + "attrs>25.4.0", # added to help pip, not a direct dep "birdhouse-birdy>=0.8.1", "black >=26.1.0", "bump-my-version >=1.2.6", From d80bb08ef22ad5fc5d4cab4663d5455e23557944 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 22 May 2026 10:19:07 -0400 Subject: [PATCH 06/15] Use conda for dev deps --- .github/workflows/main.yml | 3 ++- pyproject.toml | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c997354..3f0eb1f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,10 +73,11 @@ jobs: python=${{ matrix.python-version }} - name: Install finch-wps run: | - make develop + conda env update -f environment-dev.yml - name: Check versions run: | python -m pip check + conda list - name: Run tests run: | make start diff --git a/pyproject.toml b/pyproject.toml index fb9d37d0..cd2ed31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,6 @@ dependencies = [ [project.optional-dependencies] dev = [ - "attrs>25.4.0", # added to help pip, not a direct dep "birdhouse-birdy>=0.8.1", "black >=26.1.0", "bump-my-version >=1.2.6", From 5225cd0de61013d4f7d7047a129179a6b3df9fd2 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 22 May 2026 10:49:23 -0400 Subject: [PATCH 07/15] oups forgot to move upper python pin --- environment-dev.yml | 2 +- environment-docs.yml | 2 +- environment.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index c1a5edfd..1458c48a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python >=3.10,<3.14 + - python >=3.11,<3.15 - pywps >=4.6 - jinja2 >=3.1.4 - click >=8.1.7 diff --git a/environment-docs.yml b/environment-docs.yml index b5765d68..a49af957 100644 --- a/environment-docs.yml +++ b/environment-docs.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python >=3.11,<3.14 + - python >=3.11,<3.15 - birdy >=0.8.1 - ipython >=8.5.0 - matplotlib-base >=3.5.0 diff --git a/environment.yml b/environment.yml index 76372dcb..ff3a2c11 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python >=3.11,<3.14 + - python >=3.11,<3.15 - pip >=26.1.0 - anyascii >=0.3.0 - cftime >=1.4.1 From fc3b9b7bb9eeb5bcb46985cd0a157b7d4ce81ab7 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 22 May 2026 11:48:25 -0400 Subject: [PATCH 08/15] CI: Try to use mamba, name env, install finch --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f0eb1f2..fdf225c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,15 +69,17 @@ jobs: cache-downloads: true cache-environment: true environment-file: environment.yml + environment-name: finch create-args: >- python=${{ matrix.python-version }} - name: Install finch-wps run: | - conda env update -f environment-dev.yml + mamba env update -n finch -f environment-dev.yml + pip install --no-deps -e . - name: Check versions run: | python -m pip check - conda list + mamba list - name: Run tests run: | make start From 9e8360fccbb3d9eeb6af422af736553665579df1 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 22 May 2026 12:34:16 -0400 Subject: [PATCH 09/15] Abandon this multiple env idea - one env to rule them all. - Also micromamba doesnt contain mamba --- .github/workflows/main.yml | 3 +-- .readthedocs.yml | 2 +- Makefile | 11 ----------- docs/source/dev_guide.rst | 17 ++++++++--------- environment-dev.yml | 34 ---------------------------------- environment-docs.yml | 12 ------------ environment.yml | 31 ++++++++++++++++++++++++++++++- 7 files changed, 40 insertions(+), 70 deletions(-) delete mode 100644 environment-dev.yml delete mode 100644 environment-docs.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fdf225c4..d9d719f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,12 +74,11 @@ jobs: python=${{ matrix.python-version }} - name: Install finch-wps run: | - mamba env update -n finch -f environment-dev.yml pip install --no-deps -e . - name: Check versions run: | python -m pip check - mamba list + python -m pip list - name: Run tests run: | make start diff --git a/.readthedocs.yml b/.readthedocs.yml index 94462e57..bbf50eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -21,7 +21,7 @@ build: python: "mambaforge-23.11" conda: - environment: environment-docs.yml + environment: environment.yml # Optionally set the version of Python and requirements required to build your docs python: diff --git a/Makefile b/Makefile index 96002f6d..a215d182 100644 --- a/Makefile +++ b/Makefile @@ -38,17 +38,6 @@ help: ## print this help message. (Default) @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) ## Build targets: - -.PHONY: install -install: ## install finch application - @echo "Installing application ..." - @-bash -c 'pip install -e .' - @echo "\nStart service with \`make start\` and stop with \`make stop\`." - -develop: ## install finch application with development libraries - @echo "Installing development requirements for tests and docs ..." - @-bash -c 'pip install -e ".[dev]"' - start: ## start finch service as daemon (background process) @echo "Starting application ..." @-bash -c "$(APP_NAME) start -d" diff --git a/docs/source/dev_guide.rst b/docs/source/dev_guide.rst index 639362e7..cdc59eda 100644 --- a/docs/source/dev_guide.rst +++ b/docs/source/dev_guide.rst @@ -14,11 +14,12 @@ Developer Guide Building the docs ----------------- -First install dependencies for the documentation: +First install an environment with conda/mamba and finch within it. .. code-block:: shell - $ make develop + $ mamba env create -f environment.yml + $ pip install -e . Run the Sphinx docs generator: @@ -33,14 +34,12 @@ Running tests Run tests using pytest_. -First activate the ``finch`` Conda environment and install ``pytest``. +First install an environment with conda/mamba and finch within it. .. code-block:: shell - $ source activate finch - $ pip install -e ".[dev]" # if not already installed - # or - $ make develop + $ mamba env create -f environment.yml + $ pip install -e . Run quick tests (skip slow and online): @@ -83,9 +82,9 @@ To update the `conda` specification file for building identical environments_ on .. code-block:: console $ conda env create -f environment.yml - $ source activate finch + $ conda activate finch $ make clean - $ make install + $ pip install -e . $ conda list -n finch --explicit > spec-file.txt .. _environments: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments diff --git a/environment-dev.yml b/environment-dev.yml deleted file mode 100644 index 1458c48a..00000000 --- a/environment-dev.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: finch -channels: - - conda-forge - - nodefaults -dependencies: - - python >=3.11,<3.15 - - pywps >=4.6 - - jinja2 >=3.1.4 - - click >=8.1.7 - - psutil >=6.0.0 - # Development - - birdy >=0.8.1 - - black >=26.1.0 - - bump-my-version >=1.2.6 - - coverage >=7.5.0 - - cruft >=2.15.0 - - flake8 >=7.3.0 - - flake8-rst-docstrings >=0.4.0 - - flit >=3.11.0,<4.0 - - geojson - - lxml - - nbsphinx >=0.9.5 - - nbval >=0.10.0 - - owslib - - pip >=25.3.0 - - pre-commit >=3.5.0 - - pylint >=3.3.3 - - pytest >=8.0.0 - - pytest-cov >=5.0.0 - - pytest-flake8 - - pytest-notebook - - ruff >=0.14.0 - - watchdog >=4.0.0 - - yamllint >=1.26.0 diff --git a/environment-docs.yml b/environment-docs.yml deleted file mode 100644 index a49af957..00000000 --- a/environment-docs.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: finch -channels: - - conda-forge - - nodefaults -dependencies: - - python >=3.11,<3.15 - - birdy >=0.8.1 - - ipython >=8.5.0 - - matplotlib-base >=3.5.0 - - nbsphinx >=0.9.8 - - sphinx >=7.0.0 - - sphinxcontrib-bibtex >=2.6.0 diff --git a/environment.yml b/environment.yml index ff3a2c11..df91d1ec 100644 --- a/environment.yml +++ b/environment.yml @@ -8,7 +8,7 @@ dependencies: - anyascii >=0.3.0 - cftime >=1.4.1 - cf_xarray >=0.9.3 - - click >=8.0.0 + - click >=8.1.7 - clisops >=0.16.2 - dask >=2023.5.1 - distributed @@ -35,3 +35,32 @@ dependencies: - xscen =0.15.0 # remember to match xscen version in pyproject.toml as well # Temporary fixes - fastprogress <1.1 # FIXME: Temporary fix following https://github.com/intake/intake-esm/pull/773. Remove when intake-esm is updated in xscen. + # Dev + - birdy >=0.8.1 + - black >=26.1.0 + - bump-my-version >=1.2.6 + - coverage >=7.5.0 + - cruft >=2.15.0 + - flake8 >=7.3.0 + - flake8-rst-docstrings >=0.4.0 + - flit >=3.11.0,<4.0 + - geojson + - lxml + - nbsphinx >=0.9.8 + - nbval >=0.10.0 + - owslib + - pip >=25.3.0 + - pre-commit >=3.5.0 + - pylint >=3.3.3 + - pytest >=8.0.0 + - pytest-cov >=5.0.0 + - pytest-flake8 + - pytest-notebook + - ruff >=0.14.0 + - watchdog >=4.0.0 + - yamllint >=1.26.0 + # Docs + - ipython >=8.5.0 + - matplotlib-base >=3.5.0 + - sphinx >=7.0.0 + - sphinxcontrib-bibtex >=2.6.0 From fe5f1b5aecb7f62a3d1644d1c1dd6eb1d5d102ed Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Fri, 22 May 2026 13:03:55 -0400 Subject: [PATCH 10/15] Update a few details in docs and tooling --- .readthedocs.yml | 2 -- docs/source/installation.rst | 13 +++---------- pyproject.toml | 7 ++++--- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index bbf50eba..aa713369 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -28,5 +28,3 @@ python: install: - method: pip path: . - extra_requirements: - - docs diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 5b18e6e8..84db2955 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -39,12 +39,12 @@ Check out code from the Finch GitHub repo and start the installation: $ git clone https://github.com/bird-house/finch.git $ cd finch -Create Conda environment named `finch`: +Create Conda environment named `finch`, this includes the development and documentation dependencies : .. code-block:: console $ conda env create -f environment.yml - $ source activate finch + $ conda activate finch Install `finch` app: @@ -54,17 +54,10 @@ Install `finch` app: OR $ make install -For development you can use this command: - -.. code-block:: console - - $ pip install -e .[dev] - OR - $ make develop Install from Conda ------------------ .. note:: - `finch` is not yet available on conda-forge. But we are working on making this package available soon! + `finch` is not yet available on conda-forge. We are not working on making this package available soon. diff --git a/pyproject.toml b/pyproject.toml index cd2ed31d..c307aa74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,9 +120,10 @@ finch = "finch.cli:cli" [tool.black] target-version = [ - "py310", "py311", - "py312" + "py312", + "py313", + "py314" ] [tool.bumpversion] @@ -226,7 +227,7 @@ exclude = [ append_only = true known_first_party = "finch,_common,_utils" profile = "black" -py_version = 310 +py_version = 311 [tool.mypy] files = "." From 180d9d250f3ca0c65ab4de620b57d406e34a8cdb Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Mon, 25 May 2026 17:01:39 -0400 Subject: [PATCH 11/15] Put convenience make ruls back - increase sphinx pin - edit dev guide for coherence --- Makefile | 12 ++++++++++++ docs/source/dev_guide.rst | 8 +++++--- environment.yml | 2 +- pyproject.toml | 4 ++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a215d182..e58bf5d2 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,18 @@ help: ## print this help message. (Default) @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) ## Build targets: + +.PHONY: install +install: ## install finch application + @echo "Installing application ..." + @-bash -c 'pip install -e .' + @echo "\nStart service with \`make start\` and stop with \`make stop\`." + +develop: ## install finch application with development libraries + @echo "Installing development requirements for tests and docs ..." + @-bash -c 'test "${CONDA_PREFIX:-''} == $(dirname $(dirname $(which python)))" && echo "You are installing deps with pip inside a conda environment, this could lead to broken conda environments."' + @-bash -c 'pip install -e ".[dev]"' + start: ## start finch service as daemon (background process) @echo "Starting application ..." @-bash -c "$(APP_NAME) start -d" diff --git a/docs/source/dev_guide.rst b/docs/source/dev_guide.rst index cdc59eda..be17b814 100644 --- a/docs/source/dev_guide.rst +++ b/docs/source/dev_guide.rst @@ -19,6 +19,7 @@ First install an environment with conda/mamba and finch within it. .. code-block:: shell $ mamba env create -f environment.yml + $ mamba activate finch $ pip install -e . Run the Sphinx docs generator: @@ -39,6 +40,7 @@ First install an environment with conda/mamba and finch within it. .. code-block:: shell $ mamba env create -f environment.yml + $ mamba activate finch $ pip install -e . Run quick tests (skip slow and online): @@ -81,11 +83,11 @@ To update the `conda` specification file for building identical environments_ on .. code-block:: console - $ conda env create -f environment.yml - $ conda activate finch + $ mamba env create -f environment.yml + $ mamba activate finch $ make clean $ pip install -e . - $ conda list -n finch --explicit > spec-file.txt + $ mamba list -n finch --explicit > spec-file.txt .. _environments: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments diff --git a/environment.yml b/environment.yml index df91d1ec..21f2da6f 100644 --- a/environment.yml +++ b/environment.yml @@ -62,5 +62,5 @@ dependencies: # Docs - ipython >=8.5.0 - matplotlib-base >=3.5.0 - - sphinx >=7.0.0 + - sphinx >=9.0.0 - sphinxcontrib-bibtex >=2.6.0 diff --git a/pyproject.toml b/pyproject.toml index c307aa74..e0e581db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,11 +96,11 @@ docs = [ "birdhouse-birdy>=0.8.1", "cf-xarray>=0.9.3", "cftime>=1.4.1", - "ipython>=8.0.0", + "ipython>=8.5.0", "jupyter_client", "matplotlib>=3.5.0", "nbsphinx>=0.9.8", - "sphinx>=8.2.0", + "sphinx>=9.0.0", "sphinxcontrib-bibtex>=2.6.0" ] prod = [ From bcd7dd61459fccbf4aa1c61e2c1905f4b06acffd Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 26 May 2026 11:28:53 -0400 Subject: [PATCH 12/15] Apply suggestions from code review Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- .github/workflows/main.yml | 4 ++-- docs/source/installation.rst | 2 +- environment.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9d719f9..e00b9b73 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,11 +74,11 @@ jobs: python=${{ matrix.python-version }} - name: Install finch-wps run: | - pip install --no-deps -e . + python -m pip install --no-deps --editable . - name: Check versions run: | + micromamba list python -m pip check - python -m pip list - name: Run tests run: | make start diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 84db2955..f66885f9 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -39,7 +39,7 @@ Check out code from the Finch GitHub repo and start the installation: $ git clone https://github.com/bird-house/finch.git $ cd finch -Create Conda environment named `finch`, this includes the development and documentation dependencies : +Create Conda environment named `finch` (including the development and documentation dependencies): .. code-block:: console diff --git a/environment.yml b/environment.yml index 21f2da6f..c1ade752 100644 --- a/environment.yml +++ b/environment.yml @@ -32,7 +32,7 @@ dependencies: - xclim >=0.61,<0.62 # remember to match xclim version in pyproject.toml as well - xesmf >=0.8.2,!=0.8.8 - xsdba =0.6.1 # remember to match xsdba version in pyproject.toml as well - - xscen =0.15.0 # remember to match xscen version in pyproject.toml as well + - xscen >=0.15.0,<0.16.0 # remember to match xscen version in pyproject.toml as well # Temporary fixes - fastprogress <1.1 # FIXME: Temporary fix following https://github.com/intake/intake-esm/pull/773. Remove when intake-esm is updated in xscen. # Dev From b663e2bb5b8a9a6e170a812d0845c608833dfd3b Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 26 May 2026 13:17:55 -0400 Subject: [PATCH 13/15] Rm debug cli call - rm numcodecs and pytest-notebooks - use make in CI - no need to print env twice - make commands more explicit --- .github/workflows/main.yml | 3 +-- .readthedocs.yml | 2 ++ Makefile | 4 ++-- environment.yml | 2 -- pyproject.toml | 2 -- src/finch/cli.py | 4 ---- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e00b9b73..007b6082 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,10 +74,9 @@ jobs: python=${{ matrix.python-version }} - name: Install finch-wps run: | - python -m pip install --no-deps --editable . + make install - name: Check versions run: | - micromamba list python -m pip check - name: Run tests run: | diff --git a/.readthedocs.yml b/.readthedocs.yml index aa713369..bbf50eba 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -28,3 +28,5 @@ python: install: - method: pip path: . + extra_requirements: + - docs diff --git a/Makefile b/Makefile index e58bf5d2..87e92e2a 100644 --- a/Makefile +++ b/Makefile @@ -42,13 +42,13 @@ help: ## print this help message. (Default) .PHONY: install install: ## install finch application @echo "Installing application ..." - @-bash -c 'pip install -e .' + @-bash -c 'python -m pip install --editable .' @echo "\nStart service with \`make start\` and stop with \`make stop\`." develop: ## install finch application with development libraries @echo "Installing development requirements for tests and docs ..." @-bash -c 'test "${CONDA_PREFIX:-''} == $(dirname $(dirname $(which python)))" && echo "You are installing deps with pip inside a conda environment, this could lead to broken conda environments."' - @-bash -c 'pip install -e ".[dev]"' + @-bash -c 'python -m pip install --editable ".[dev]"' start: ## start finch service as daemon (background process) @echo "Starting application ..." diff --git a/environment.yml b/environment.yml index c1ade752..85ba4daf 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,6 @@ dependencies: - geopandas >=1.0 - jinja2 >=3.1.4 - netcdf4 - - numcodecs - numpy >=1.25.0 - pandas >=2.2.0,<3.0 - parse >=1.20 @@ -55,7 +54,6 @@ dependencies: - pytest >=8.0.0 - pytest-cov >=5.0.0 - pytest-flake8 - - pytest-notebook - ruff >=0.14.0 - watchdog >=4.0.0 - yamllint >=1.26.0 diff --git a/pyproject.toml b/pyproject.toml index e0e581db..330f33ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ "geopandas >=1.0", "jinja2 >=3.1.4", "netcdf4", - "numcodecs", "numpy >=1.25.0", "pandas >=2.2.0,<3.0", "parse >=1.20", @@ -86,7 +85,6 @@ dev = [ "pytest >=8.0.0", "pytest-cov >=5.0.0", "pytest-flake8", - "pytest-notebook", "ruff >=0.14.0", "vulture >=2.14", "watchdog >=4.0.0" diff --git a/src/finch/cli.py b/src/finch/cli.py index 3c2e2cf4..e2292de1 100644 --- a/src/finch/cli.py +++ b/src/finch/cli.py @@ -301,7 +301,3 @@ def start( else: # no daemon _run(app, bind_host=bind_host) - - -if __name__ == "__main__": - cli() From f3e94a05f8a457d84ebc163d374a491e56f87eec Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 26 May 2026 13:21:29 -0400 Subject: [PATCH 14/15] Forgot to upd xscen pins in pyproject --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 85ba4daf..2dc1d727 100644 --- a/environment.yml +++ b/environment.yml @@ -31,7 +31,7 @@ dependencies: - xclim >=0.61,<0.62 # remember to match xclim version in pyproject.toml as well - xesmf >=0.8.2,!=0.8.8 - xsdba =0.6.1 # remember to match xsdba version in pyproject.toml as well - - xscen >=0.15.0,<0.16.0 # remember to match xscen version in pyproject.toml as well + - xscen >=0.15,<0.16 # remember to match xscen version in pyproject.toml as well # Temporary fixes - fastprogress <1.1 # FIXME: Temporary fix following https://github.com/intake/intake-esm/pull/773. Remove when intake-esm is updated in xscen. # Dev diff --git a/pyproject.toml b/pyproject.toml index 330f33ca..bee6b769 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "xclim >=0.61,<0.62", # remember to match xclim version in environment.yml as well "xesmf >=0.8.2,!=0.8.8", "xsdba ==0.6.1", # remember to match xsdba version in environment.yml as well - "xscen ==0.15.0", # remember to match xscen version in environment.yml as well + "xscen >=0.15,<0.16", # remember to match xscen version in environment.yml as well "werkzeug>=3.0.6", # Temporary fixes "fastprogress <1.1" # FIXME: Temporary fix following https://github.com/intake/intake-esm/pull/773. Remove when intake-esm is updated in xscen. From 816072e7f5b2a69ef9f4a2015bc86772152dd5c8 Mon Sep 17 00:00:00 2001 From: Pascal Bourgault Date: Tue, 26 May 2026 14:53:09 -0400 Subject: [PATCH 15/15] Update docs/source/installation.rst Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f66885f9..7ee6c09e 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -60,4 +60,4 @@ Install from Conda .. note:: - `finch` is not yet available on conda-forge. We are not working on making this package available soon. + There are no plans to make `finch` available on conda-forge.