diff --git a/.github/markdown-link-check-config.json b/.github/markdown-link-check-config.json new file mode 100644 index 0000000..40b883e --- /dev/null +++ b/.github/markdown-link-check-config.json @@ -0,0 +1,31 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://localhost" + }, + { + "pattern": "^https://127.0.0.1" + }, + { + "pattern": "^file://" + } + ], + "replacementPatterns": [ + { + "pattern": "^/", + "replacement": "{{BASEURL}}/" + } + ], + "httpHeaders": [ + { + "urls": ["https://github.com"], + "headers": { + "Accept": "text/html" + } + } + ], + "timeout": "20s", + "retryOn429": true, + "retryCount": 3, + "fallbackHttpStatus": [400, 401, 403, 404, 405, 500, 502, 503, 504] +} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1aea861 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,117 @@ +name: CI + +on: + push: + branches: [ master, main, develop ] + pull_request: + branches: [ master, main ] + workflow_dispatch: + +jobs: + test-python: + name: Test Python Code + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + + - name: Check Python scripts syntax + run: | + pixi run python -m py_compile run_ocp_tool.py + pixi run python -m py_compile ocp_tool/*.py + echo "✓ All Python scripts have valid syntax" + + - name: Test import of ocp_tool modules + run: | + pixi run python -c " + from ocp_tool.config import load_config, OCPConfig + from ocp_tool.gaussian_grids import generate_gaussian_grid + from ocp_tool.lsm import process_land_sea_mask + from ocp_tool.oasis_writer import write_oasis_grid_files + from ocp_tool.runoff import modify_runoff_map + from ocp_tool.plotting import plot_land_sea_mask + print('✓ Successfully imported all ocp_tool modules') + " + + code-quality: + name: Code Quality Checks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + environments: dev + + - name: Run flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + pixi run -e dev flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,build,dist,.pixi + # Exit-zero treats all errors as warnings + pixi run -e dev flake8 . --count --exit-zero --max-complexity=15 --max-line-length=120 --statistics --exclude=.git,__pycache__,build,dist,.pixi + + - name: Check code formatting with black + run: | + pixi run -e dev black --check --diff --exclude='(\.git|\.pixi)' . || echo "::warning::Code formatting issues found." + + - name: Check import sorting with isort + run: | + pixi run -e dev isort --check-only --diff --skip .pixi . || echo "::warning::Import sorting issues found." + + documentation: + name: Documentation Check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check README files + run: | + for readme in README.md workflow/README.md; do + if [ -f "$readme" ]; then + echo "✓ Found $readme" + else + echo "::error::Missing $readme" + exit 1 + fi + done + + - name: Check for broken links in documentation + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + config-file: '.github/markdown-link-check-config.json' + continue-on-error: true + + security-scan: + name: Security Scanning + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy security scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + continue-on-error: true + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: 'trivy-results.sarif' + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..6497db5 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,45 @@ +name: Documentation + +on: + push: + branches: [ master, main ] + paths: + - 'docs/**' + - 'ocp_tool/**' + pull_request: + branches: [ master, main ] + paths: + - 'docs/**' + - 'ocp_tool/**' + workflow_dispatch: + +jobs: + build-docs: + name: Build Documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install sphinx alabaster numpy scipy xarray netCDF4 pyyaml matplotlib cartopy + pip install -e . + + - name: Build documentation + run: | + cd docs + sphinx-build -b html source build/html + + - name: Upload documentation artifact + uses: actions/upload-artifact@v4 + with: + name: documentation + path: docs/build/html/ + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c0d24a6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,211 @@ +name: Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Release tag' + required: true + type: string + +jobs: + create-release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + version: ${{ steps.get_version.outputs.version }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version + id: get_version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.tag }}" + else + VERSION=${GITHUB_REF#refs/tags/} + fi + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + - name: Generate changelog + id: changelog + run: | + # Generate changelog from git commits since last tag + LAST_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -z "$LAST_TAG" ]; then + CHANGES=$(git log --pretty=format:"- %s (%h)" --no-merges) + else + CHANGES=$(git log ${LAST_TAG}..HEAD --pretty=format:"- %s (%h)" --no-merges) + fi + + cat > CHANGELOG.md << EOF + # Changes in ${{ steps.get_version.outputs.version }} + + ${CHANGES} + + ## Full Changelog + + See [commits](https://github.com/${{ github.repository }}/compare/${LAST_TAG}...${{ steps.get_version.outputs.version }}) for full details. + EOF + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.get_version.outputs.version }} + release_name: Release ${{ steps.get_version.outputs.version }} + body_path: CHANGELOG.md + draft: false + prerelease: ${{ contains(steps.get_version.outputs.version, '-') }} + + build-release-containers: + name: Build Release Containers + needs: create-release + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + target: [production, development] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: | + suffix=-${{ matrix.target }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push release image + uses: docker/build-push-action@v5 + with: + context: . + file: workflow/containers/Dockerfile + target: ${{ matrix.target }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + build-singularity-release: + name: Build Singularity Release + needs: create-release + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Singularity + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.11.4 + + - name: Build Singularity container + run: | + sudo singularity build ocp-tool-${{ needs.create-release.outputs.version }}.sif workflow/containers/singularity.def + + - name: Upload Singularity image to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./ocp-tool-${{ needs.create-release.outputs.version }}.sif + asset_name: ocp-tool-${{ needs.create-release.outputs.version }}.sif + asset_content_type: application/octet-stream + + upload-workflow-assets: + name: Upload Workflow Assets + needs: create-release + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create workflow archive + run: | + tar -czf ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz \ + workflow/ \ + environment.yaml \ + README.md \ + --exclude='workflow/logs' \ + --exclude='workflow/temp' + + - name: Upload workflow archive to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz + asset_name: ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz + asset_content_type: application/gzip + + notify-release: + name: Notify Release + needs: [create-release, build-release-containers, build-singularity-release] + runs-on: ubuntu-latest + if: always() + + steps: + - name: Create release summary + run: | + cat > $GITHUB_STEP_SUMMARY << EOF + # 🚀 Release ${{ needs.create-release.outputs.version }} Complete + + ## 📦 Available Assets + + - **Docker Images**: \`ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production\` + - **Development Image**: \`ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-development\` + - **Singularity Container**: \`ocp-tool-${{ needs.create-release.outputs.version }}.sif\` + - **Workflow Archive**: \`ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz\` + + ## 🐳 Quick Start with Docker + + \`\`\`bash + docker pull ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production + docker run --rm -v \$(pwd):/app/data ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production snakemake --cores 4 + \`\`\` + + ## 📖 Documentation + + Visit the [documentation](https://${{ github.repository_owner }}.github.io/ocp-tool/) for usage guides and API reference. + EOF \ No newline at end of file diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/_templates/.gitkeep b/docs/source/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..e67b0f0 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,50 @@ +API Reference +============= + +Configuration +------------- + +.. automodule:: ocp_tool.config + :members: + :undoc-members: + :show-inheritance: + +Grid Processing +--------------- + +.. automodule:: ocp_tool.gaussian_grids + :members: + :undoc-members: + :show-inheritance: + +Land-Sea Mask +------------- + +.. automodule:: ocp_tool.lsm + :members: + :undoc-members: + :show-inheritance: + +OASIS Writer +------------ + +.. automodule:: ocp_tool.oasis_writer + :members: + :undoc-members: + :show-inheritance: + +Runoff Processing +----------------- + +.. automodule:: ocp_tool.runoff + :members: + :undoc-members: + :show-inheritance: + +Plotting +-------- + +.. automodule:: ocp_tool.plotting + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..881aa1d --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,25 @@ +"""Sphinx configuration for OCP-Tool documentation.""" + +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + +project = 'OCP-Tool' +copyright = '2024, AWI Climate Dynamics' +author = 'Jan Streffing' + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +html_theme = 'alabaster' +html_static_path = ['_static'] + +# Napoleon settings for Google/NumPy style docstrings +napoleon_google_docstring = True +napoleon_numpy_docstring = True diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..47f0a32 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +OCP-Tool Documentation +====================== + +OpenIFS Coupling Preparation Tool for climate model coupling. + +OCP-Tool generates OASIS3-MCT input files, modifies the OpenIFS land-sea mask, +and adjusts runoff maps to fit a given FESOM2 mesh or NEMO grid. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + usage + api + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 0000000..0863459 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,43 @@ +Installation +============ + +Requirements +------------ + +- Python 3.9+ +- NetCDF libraries +- ECCODES for GRIB processing +- CDO (Climate Data Operators) +- NCO (NetCDF Operators) + +Quick Install (Recommended) +--------------------------- + +Clone from GitHub and install dependencies manually:: + + git clone https://github.com/JanStreffing/ocp-tool.git + cd ocp-tool + +Then install the required Python packages in your existing environment. +This is the most common approach, especially on HPC systems where +you may already have the dependencies available via modules. + +Using conda/mamba +----------------- + +Create a dedicated environment with all dependencies:: + + git clone https://github.com/JanStreffing/ocp-tool.git + cd ocp-tool + conda env create -f environment.yaml + conda activate ocp-tool + pip install -e . + +Using pixi +---------- + +Pixi handles all dependencies automatically:: + + git clone https://github.com/JanStreffing/ocp-tool.git + cd ocp-tool + pixi install diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..4c58d73 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,47 @@ +Usage +===== + +Configuration +------------- + +OCP-Tool uses YAML configuration files. Example configs are in the ``configs/`` directory: + +- ``TCO95_CORE2.yaml`` - TCO95 atmosphere with CORE2 ocean mesh +- ``TCO319_CORE3.yaml`` - TCO319 atmosphere with CORE3 ocean mesh (with ice cavities) + +Running the Tool +---------------- + +Basic usage:: + + python run_ocp_tool.py configs/TCO95_CORE2.yaml + +Configuration Options +--------------------- + +Atmosphere settings:: + + atmosphere: + resolution_list: [95] # TCO95 + truncation_type: "cubic-octahedral" + experiment_name: "ab45" # ICMGG file prefix + +Ocean settings:: + + ocean: + grid_name: "CORE2" + has_ice_cavities: false + mesh_file: "/path/to/mesh.nc" + +Output Structure +---------------- + +Output is organized by grid combination:: + + output/ + └── TCO95_CORE2/ + ├── lpj-guess/ + ├── oasis_mct3_input/ + ├── openifs_input_modified/ + ├── plots/ + └── runoff_map_modified/ diff --git a/ocp_tool/config.py b/ocp_tool/config.py index 0740bdc..08446a1 100644 --- a/ocp_tool/config.py +++ b/ocp_tool/config.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Union import yaml @@ -23,6 +23,8 @@ class OceanConfig: grid_name: str has_ice_cavities: bool mesh_file: Path + intermediate_resolution: str = "r360x181" + force_overwrite_griddes: bool = False @dataclass @@ -96,7 +98,7 @@ def get_icmgg_iniua_file(self) -> Path: return self.output_paths.openifs_modified / f'ICMGG{self.atmosphere.experiment_name}INIUA' -def load_config(config_path: str | Path) -> OCPConfig: +def load_config(config_path: Union[str, Path]) -> OCPConfig: """ Load configuration from YAML file. @@ -170,6 +172,8 @@ def resolve_path(path_str: str) -> Path: grid_name=raw['ocean']['grid_name'], has_ice_cavities=raw['ocean']['has_ice_cavities'], mesh_file=Path(raw['ocean']['mesh_file']), + intermediate_resolution=raw['ocean'].get('intermediate_resolution', 'r360x181'), + force_overwrite_griddes=raw['ocean'].get('force_overwrite_griddes', False), ), runoff=RunoffConfig( manual_basin_removal=raw['runoff']['manual_basin_removal'], diff --git a/ocp_tool/create_outputdirs.py b/ocp_tool/create_outputdirs.py new file mode 100644 index 0000000..0427a40 --- /dev/null +++ b/ocp_tool/create_outputdirs.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from os import makedirs + +def create_outputdirs(config, resolution): + """Create necessary output directories based on configuration and resolution.""" + output_dir = f"./output/TCO{resolution}_{config.ocean.grid_name}/" + makedirs(f"{output_dir}lpj-guess", exist_ok=True) + makedirs(f"{output_dir}oasis_mct3_input", exist_ok=True) + makedirs(f"{output_dir}openifs_input_modified", exist_ok=True) + makedirs(f"{output_dir}plots", exist_ok=True) + makedirs(f"{output_dir}runoff_map_modified", exist_ok=True) \ No newline at end of file diff --git a/ocp_tool/gaussian_grids.py b/ocp_tool/gaussian_grids.py index f391c08..2785466 100644 --- a/ocp_tool/gaussian_grids.py +++ b/ocp_tool/gaussian_grids.py @@ -361,6 +361,9 @@ def read_fesom_grid_polygon( falls inside the FESOM ocean mesh. It scales well for high-resolution grids as it doesn't require an intermediate regular grid. + If mesh_file doesn't exist or force_overwrite_griddes is True, attempts to + generate mesh.nc from FESOM ASCII files using pyfesom2. + Algorithm: 1. Build ocean boundary from coastal edges + calving front edges 2. Create convex hulls for small disconnected cavity components @@ -385,6 +388,25 @@ def read_fesom_grid_polygon( mesh_file = config.ocean.mesh_file has_cavities = config.ocean.has_ice_cavities + # Generate mesh.nc from ASCII files if needed + if not mesh_file.exists() or config.ocean.force_overwrite_griddes: + try: + import pyfesom2 as pf + griddir = str(mesh_file.parent) + if mesh_file.exists(): + print(f" mesh.nc exists but force_overwrite_griddes=True, regenerating via pyfesom2") + else: + print(f" mesh.nc not found, generating from ASCII files via pyfesom2") + print(f" Reading FESOM ASCII grid from: {griddir}") + fesom_grid = pf.read_fesom_ascii_grid(griddir=griddir, cavity=has_cavities) + pf.write_mesh_to_netcdf(fesom_grid, ofile=str(mesh_file), overwrite=True, cavity=has_cavities) + print(f" Created mesh.nc: {mesh_file}") + except ImportError: + raise ImportError("pyfesom2 is required for mesh generation. " + "Install with: pip install git+https://github.com/FESOM/pyfesom2.git") + except Exception as e: + raise RuntimeError(f"Failed to generate mesh.nc via pyfesom2: {e}") + print(f" Loading FESOM mesh: {mesh_file}") mesh = Dataset(str(mesh_file), 'r') lon = mesh.variables['lon'][:] @@ -392,8 +414,9 @@ def read_fesom_grid_polygon( triag = mesh.variables['triag_nodes'][:] - 1 # Convert to 0-indexed if has_cavities and 'cav_nod_mask' in mesh.variables: - cav_mask = mesh.variables['cav_nod_mask'][:] - print(f" Cavity nodes: {int(np.sum(cav_mask == 1))}") + cav_mask = np.ma.filled(mesh.variables['cav_nod_mask'][:], 0) + n_cavity = int(np.sum(cav_mask == 1)) + print(f" Cavity nodes: {n_cavity}") else: cav_mask = np.zeros(len(lon)) print(" No cavity mask found or cavities disabled") diff --git a/ocp_tool/grids.py b/ocp_tool/grids.py new file mode 100644 index 0000000..e69de29 diff --git a/ocp_tool/lsm.py b/ocp_tool/lsm.py index cc7006b..6856cb0 100644 --- a/ocp_tool/lsm.py +++ b/ocp_tool/lsm.py @@ -157,6 +157,7 @@ def modify_lsm( if ocean_grid_name != 'AMIP': # Automatic lake removal based on ocean mask + # Polygon method: ocean_lsm = 1 means land, 0 means ocean n_points = len(gribfield_mod[slt_id]) for i in range(n_points - 1): diff --git a/ocp_tool/oasis_writer.py b/ocp_tool/oasis_writer.py index 892b455..53410de 100644 --- a/ocp_tool/oasis_writer.py +++ b/ocp_tool/oasis_writer.py @@ -96,8 +96,15 @@ def _write_single_oasis_file( lonname = f'{grids_name}.lon' latname = f'{grids_name}.lat' - nc.createDimension(xname, grid.center_lons.shape[1]) - nc.createDimension(yname, 1) + # Only create dimensions if they don't exist + if xname not in nc.dimensions: + nc.createDimension(xname, grid.center_lons.shape[1]) + if yname not in nc.dimensions: + nc.createDimension(yname, 1) + + # Skip if variables already exist (avoid duplicates) + if lonname in nc.variables: + continue id_lon = nc.createVariable(lonname, 'float64', (yname, xname)) id_lat = nc.createVariable(latname, 'float64', (yname, xname)) @@ -111,7 +118,8 @@ def _write_single_oasis_file( crnname = f'crn_{grids_name}' cloname = f'{grids_name}.clo' claname = f'{grids_name}.cla' - nc.createDimension(crnname, 4) + if crnname not in nc.dimensions: + nc.createDimension(crnname, 4) id_clo = nc.createVariable(cloname, 'float64', (crnname, yname, xname)) id_cla = nc.createVariable(claname, 'float64', (crnname, yname, xname)) diff --git a/ocp_tool/plotting.py b/ocp_tool/plotting.py index f936450..a8d83e6 100644 --- a/ocp_tool/plotting.py +++ b/ocp_tool/plotting.py @@ -1,6 +1,7 @@ """ Plotting module for OCP-Tool. Handles visualization of land-sea masks and runoff maps. +Uses polygon-based plotting with cell corners for accurate representation. """ from pathlib import Path @@ -8,13 +9,47 @@ import numpy as np import matplotlib.pyplot as plt -from mpl_toolkits.basemap import Basemap +from matplotlib.collections import PolyCollection +import cartopy.crs as ccrs +import cartopy.feature as cfeature from .config import OCPConfig from .gaussian_grids import GaussianGrid from .lsm import LSMData +def _build_cell_vertices(grid: GaussianGrid) -> np.ndarray: + """ + Build cell vertices array from grid corner coordinates. + + Args: + grid: GaussianGrid with corner_lons and corner_lats + + Returns: + Array of shape (n_cells, 4, 2) with [lon, lat] for each corner + """ + # corner_lons/lats have shape (4, 1, n_points), squeeze and transpose + clo = grid.corner_lons.squeeze().T # (n_points, 4) + cla = grid.corner_lats.squeeze().T # (n_points, 4) + cell_verts = np.stack([clo, cla], axis=-1) # (n_points, 4, 2) + return cell_verts + + +def _get_valid_cell_mask(grid: GaussianGrid) -> np.ndarray: + """ + Create mask for cells that don't wrap around the date line. + + Args: + grid: GaussianGrid with corner coordinates + + Returns: + Boolean mask array + """ + clo = grid.corner_lons.squeeze().T # (n_points, 4) + lon_range = np.max(clo, axis=1) - np.min(clo, axis=1) + return lon_range < 180 + + def plot_land_sea_mask( config: OCPConfig, grid: GaussianGrid, @@ -22,32 +57,75 @@ def plot_land_sea_mask( resolution: int ) -> None: """ - Plot the final land-sea mask showing wet and dry points. + Plot the final land-sea mask using polygon cells. Args: config: OCP configuration - grid: Gaussian grid data + grid: Gaussian grid data with corners lsm_data: Land-sea mask data resolution: Truncation number """ - fig = plt.figure(figsize=(24, 14)) - ax = fig.add_subplot(111) + fig = plt.figure(figsize=(16, 10)) + ax = fig.add_subplot(111, projection=ccrs.PlateCarree()) + + # Build cell vertices from corners + cell_verts = _build_cell_vertices(grid) + valid_mask = _get_valid_cell_mask(grid) - # Extract wet and dry points - lsm_atm = lsm_data.lsm_binary_atm - lsm_land = lsm_data.lsm_binary_land + # Get LSM values (flatten to 1D) + lsm_atm = lsm_data.lsm_binary_atm.flatten() + lsm_land = lsm_data.lsm_binary_land.flatten() - xpts_atm = grid.center_lons[np.round(lsm_atm[:, :]) < 1] - ypts_atm = grid.center_lats[np.round(lsm_atm[:, :]) < 1] - xpts_land = grid.center_lons[np.round(lsm_land[:, :]) < 1] - ypts_land = grid.center_lats[np.round(lsm_land[:, :]) < 1] + # Create color array: 0=wet (blue), 1=land (tan), new dry points (red) + # wet points: lsm_atm < 1 + # new dry points: lsm_land < 1 but lsm_atm >= 1 (points that became dry) + colors = np.zeros(len(lsm_atm)) + colors[lsm_atm >= 0.5] = 1 # Land + colors[lsm_atm < 0.5] = 0 # Wet - ax.scatter(xpts_land, ypts_land, s=100/resolution, color='red', marker='.', label='New dry points') - ax.scatter(xpts_atm, ypts_atm, s=200/resolution, marker='.', label='Wet points') - ax.legend(loc="lower right") + # Identify new dry points (were wet in atm, now dry in land) + new_dry = (np.round(lsm_land) < 1) & (np.round(lsm_atm) >= 0.5) + colors[new_dry] = 2 # New dry points + + # Apply valid mask + valid_verts = cell_verts[valid_mask] + valid_colors = colors[valid_mask] + + # Create custom colormap: blue=wet, tan=land, red=new dry + from matplotlib.colors import ListedColormap + cmap = ListedColormap(['#4169E1', '#D2B48C', '#FF4444']) # Blue, Tan, Red + + # Create PolyCollection + collection = PolyCollection( + valid_verts, + array=valid_colors, + cmap=cmap, + edgecolors='none', + linewidths=0, + transform=ccrs.PlateCarree(), + ) + collection.set_clim(0, 2) + ax.add_collection(collection) + + # Set global extent + ax.set_global() + + # Add coastlines for reference + ax.coastlines(linewidth=0.5, color='black', zorder=5) + + # Legend + from matplotlib.patches import Patch + legend_elements = [ + Patch(facecolor='#4169E1', label='Wet points'), + Patch(facecolor='#D2B48C', label='Land'), + Patch(facecolor='#FF4444', label='New dry points'), + ] + ax.legend(handles=legend_elements, loc='lower right') + + ax.set_title(f'Land-Sea Mask T{resolution}', fontsize=14) output_file = config.output_paths.plots / f'land_points_T{resolution}.png' - fig.savefig(str(output_file), format='png', dpi=600) + fig.savefig(str(output_file), format='png', dpi=300, bbox_inches='tight') plt.close(fig) print(f"Saved LSM plot to {output_file}") @@ -81,45 +159,35 @@ def plot_runoff_maps( cmap = plt.cm.flag - # Amazon region - m = Basemap( - llcrnrlon=-60., llcrnrlat=-10, - urcrnrlon=-30., urcrnrlat=20., - resolution='l', area_thresh=1000., projection='cyl' - ) - xi, yi = m(lon, lat) - + # Amazon region - arrival fig = plt.figure(figsize=(12, 8)) - m.pcolor(xi, yi, arrival_cat, cmap=cmap) - m.drawcoastlines() - m.drawparallels(np.arange(-90., 120., 45.)) - m.drawmeridians(np.arange(0., 360., 90.)) + ax = fig.add_subplot(111, projection=ccrs.PlateCarree()) + ax.set_extent([-60, -30, -10, 20], crs=ccrs.PlateCarree()) + ax.pcolormesh(lon, lat, arrival_cat, cmap=cmap, transform=ccrs.PlateCarree()) + ax.add_feature(cfeature.COASTLINE) + ax.gridlines(draw_labels=True) output_file = config.output_paths.plots / 'runoff_amazon_arrival.png' fig.savefig(str(output_file), format='png') plt.close(fig) # Ob region - drainage - m = Basemap( - llcrnrlon=50., llcrnrlat=40, - urcrnrlon=110., urcrnrlat=80., - resolution='l', area_thresh=1000., projection='cyl' - ) - fig = plt.figure(figsize=(12, 8)) - m.pcolor(xi, yi, drainage_cat, cmap=cmap) - m.drawcoastlines() - m.drawparallels(np.arange(-90., 120., 45.)) - m.drawmeridians(np.arange(0., 360., 90.)) + ax = fig.add_subplot(111, projection=ccrs.PlateCarree()) + ax.set_extent([50, 110, 40, 80], crs=ccrs.PlateCarree()) + ax.pcolormesh(lon, lat, drainage_cat, cmap=cmap, transform=ccrs.PlateCarree()) + ax.add_feature(cfeature.COASTLINE) + ax.gridlines(draw_labels=True) output_file = config.output_paths.plots / 'runoff_ob_drainage.png' fig.savefig(str(output_file), format='png') plt.close(fig) # Ob region - arrival fig = plt.figure(figsize=(12, 8)) - m.pcolor(xi, yi, arrival_cat, cmap=cmap) - m.drawcoastlines() - m.drawparallels(np.arange(-90., 120., 45.)) - m.drawmeridians(np.arange(0., 360., 90.)) + ax = fig.add_subplot(111, projection=ccrs.PlateCarree()) + ax.set_extent([50, 110, 40, 80], crs=ccrs.PlateCarree()) + ax.pcolormesh(lon, lat, arrival_cat, cmap=cmap, transform=ccrs.PlateCarree()) + ax.add_feature(cfeature.COASTLINE) + ax.gridlines(draw_labels=True) output_file = config.output_paths.plots / 'runoff_ob_arrival.png' fig.savefig(str(output_file), format='png') plt.close(fig) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..be4c22d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,178 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "ocp-tool" +description = "Tool to generate OASIS files for coupling OpenIFS, FESOM2, and NEMO" +authors = [ + {name = "Jan Streffing", email = "jan.streffing@awi.de"} +] +readme = "README.md" +license = {file = "licence"} +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: Atmospheric Science", +] +dependencies = [ + "numpy", + "netcdf4", + "eccodes", + "matplotlib", + "pandas", + "pyyaml", + "tqdm", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/pgierz/ocp-tool" +Repository = "https://github.com/pgierz/ocp-tool" +Issues = "https://github.com/pgierz/ocp-tool/issues" + +[project.entry-points."scriptengine.tasks"] +"ocpt.main" = "ocp_tool.scriptengine_task:OCPTool" + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "black", + "isort", + "flake8", + "mypy", +] +jupyter = [ + "jupyterlab", + "notebook", + "ipykernel", +] +docs = [ + "sphinx", + "sphinx-rtd-theme", + "nbsphinx", + "myst-parser", + "sphinx-autoapi", +] + +[tool.setuptools] +packages = ["ocp_tool"] + +[tool.setuptools.dynamic] +version = {attr = "ocp_tool.__version__"} + +# Pixi configuration +[tool.pixi.workspace] +name = "ocp-tool" +description = "Tool to generate OASIS files for coupling OpenIFS, FESOM2, and NEMO" +authors = ["Jan Streffing "] +channels = ["conda-forge", "bioconda", "eumetsat"] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] + +[tool.pixi.dependencies] +python = ">=3.9" +numpy = "*" +scipy = "*" +netcdf4 = "*" +xarray = "*" +matplotlib = "*" +cartopy = "*" +pandas = "*" +pyyaml = "*" +python-eccodes = "*" +pip = "*" + +[tool.pixi.pypi-dependencies] +# Use modernized pyfesom2 branch with Python 3.11+ support +pyfesom2 = { git = "https://github.com/FESOM/pyfesom2.git", branch = "modernize-packaging" } + +[tool.pixi.feature.jupyter.dependencies] +jupyterlab = "*" +notebook = "*" +ipykernel = "*" + +[tool.pixi.feature.dev.dependencies] +pytest = "*" +pytest-cov = "*" +black = "*" +isort = "*" +flake8 = "*" +mypy = "*" + +[tool.pixi.feature.docs.dependencies] +sphinx = "*" +sphinx-rtd-theme = "*" +nbsphinx = "*" +myst-parser = "*" +sphinx-autoapi = "*" + +[tool.pixi.environments] +default = { solve-group = "default" } +jupyter = { features = ["jupyter"], solve-group = "default" } +dev = { features = ["dev"], solve-group = "default" } +docs = { features = ["docs"], solve-group = "default" } +full = { features = ["jupyter", "dev", "docs"], solve-group = "default" } + +[tool.pixi.tasks] +lint = "flake8 ocp_tool" +format = "black ocp_tool" +sort-imports = "isort ocp_tool" +type-check = "mypy ocp_tool" +test = "pytest" +clean-format = { depends-on = ["format", "sort-imports"] } +jupyter = "jupyter lab" +notebook = "jupyter notebook" +docs-build = "sphinx-build -W -b html docs/source docs/build/html" +docs-clean = "rm -rf docs/build" +docs-serve = "python -m http.server 8000 --directory docs/build/html" +workflow-dry = { cmd = "snakemake --dry-run", cwd = "workflow" } +workflow-lint = { cmd = "snakemake --lint", cwd = "workflow" } +workflow-run = { cmd = "snakemake --cores 4 --use-conda", cwd = "workflow" } +build = "python -m build" +install-dev = "pip install -e ." + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["ocp_tool"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--cov=ocp_tool", + "--cov-report=term-missing", + "--cov-report=html", +] \ No newline at end of file diff --git a/run_ocp_tool.py b/run_ocp_tool.py index 260178c..17ba5b1 100644 --- a/run_ocp_tool.py +++ b/run_ocp_tool.py @@ -12,6 +12,7 @@ import sys import time +from os import makedirs from pathlib import Path from ocp_tool.config import load_config, OCPConfig @@ -22,6 +23,7 @@ from ocp_tool.plotting import plot_land_sea_mask, plot_runoff_maps from ocp_tool.co2_interpolation import interpolate_co2_to_icmgg from ocp_tool.field_interpolation import interpolate_2d_fields_to_icmgg +from ocp_tool.create_outputdirs import create_outputdirs def run_ocp_tool(config: OCPConfig) -> None: @@ -39,14 +41,19 @@ def run_ocp_tool(config: OCPConfig) -> None: print(f" Ocean grid: {config.ocean.grid_name}") print(f" Ice cavities: {config.ocean.has_ice_cavities}") print(f" Experiment: {config.atmosphere.experiment_name}") - print() + # Process each resolution for resolution in config.atmosphere.resolution_list: print(f"\n{'='*60}") print(f" Processing resolution T{resolution}") + print(f" Output: ./output/TCO{resolution}_{config.ocean.grid_name}") print(f"{'='*60}\n") + # Step 0: Create Output directories + print("Step 0: Creating output directories...") + create_outputdirs(config, resolution) + # Step 1: Generate Gaussian grid print("Step 1: Generating Gaussian grid coordinates...") grid = generate_gaussian_grid(config, resolution) diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 0000000..988ceb7 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,35 @@ +# OCP-tool Snakemake Workflow + +This directory contains the Snakemake workflow for automated climate model preparation. + +## Usage + +Configure your settings in `config/config.yaml`, then run: + +```bash +# Local execution +snakemake --cores 4 + +# With pixi +pixi run workflow-run + +# Dry run to check workflow +pixi run workflow-dry +``` + +## Configuration + +Edit `config/config.yaml` to specify: +- Grid resolution (`res_num`) +- Experiment name (`exp_name_oifs`) +- Ocean grid name (`grid_name_oce`) +- Input/output paths + +## Rules + +- `prepare_gaussian_grids`: Process OpenIFS grid files +- `process_fesom_grid`: Handle ocean model grids +- `modify_land_sea_mask`: Core LSM modification +- `generate_oasis_files`: Create OASIS3-MCT files +- `modify_runoff_maps`: Adjust runoff routing +- `generate_plots`: Create visualizations \ No newline at end of file