diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..82d6d15 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,13 @@ +{ + "permissions": { + "allow": [ + "Bash(git cherry-pick:*)", + "Bash(gh:*)", + "Bash(act:*)", + "Bash(pixi install:*)", + "Bash(git for-each-ref:*)" + ], + "deny": [], + "ask": [] + } +} \ No newline at end of file diff --git a/.github/markdown-link-check-config.json b/.github/markdown-link-check-config.json new file mode 100644 index 0000000..40b883e --- /dev/null +++ b/.github/markdown-link-check-config.json @@ -0,0 +1,31 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://localhost" + }, + { + "pattern": "^https://127.0.0.1" + }, + { + "pattern": "^file://" + } + ], + "replacementPatterns": [ + { + "pattern": "^/", + "replacement": "{{BASEURL}}/" + } + ], + "httpHeaders": [ + { + "urls": ["https://github.com"], + "headers": { + "Accept": "text/html" + } + } + ], + "timeout": "20s", + "retryOn429": true, + "retryCount": 3, + "fallbackHttpStatus": [400, 401, 403, 404, 405, 500, 502, 503, 504] +} \ No newline at end of file diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml new file mode 100644 index 0000000..5c26e0d --- /dev/null +++ b/.github/workflows/build-containers.yml @@ -0,0 +1,235 @@ +name: Build and Push Containers + +on: + push: + branches: [ master, main ] + tags: [ 'v*' ] + pull_request: + branches: [ master, main ] + workflow_dispatch: + inputs: + push_images: + description: 'Push images to registry' + required: false + default: false + type: boolean + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-docker: + name: Build Docker Images + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + target: [production, development] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: | + suffix=-${{ matrix.target }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: workflow/containers/Dockerfile + target: ${{ matrix.target }} + push: ${{ github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event.inputs.push_images == 'true') }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + test-containers: + name: Test Container Images + runs-on: ubuntu-latest + needs: build-docker + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + + strategy: + matrix: + target: [production, development] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test image + uses: docker/build-push-action@v5 + with: + context: . + file: workflow/containers/Dockerfile + target: ${{ matrix.target }} + load: true + tags: ocp-tool:test-${{ matrix.target }} + cache-from: type=gha + + - name: Test production container + if: matrix.target == 'production' + run: | + # Test basic functionality + docker run --rm ocp-tool:test-production python --version + docker run --rm ocp-tool:test-production snakemake --version + + # Test OCP-tool imports + docker run --rm ocp-tool:test-production python -c "import ocp_tool; print('✓ OCP-tool import successful')" + + # Test conda environment + docker run --rm ocp-tool:test-production conda list | grep -E "(numpy|matplotlib|netcdf4)" + + - name: Test development container + if: matrix.target == 'development' + run: | + # Test Jupyter installation + docker run --rm ocp-tool:test-development jupyter --version + docker run --rm ocp-tool:test-development jupyter kernelspec list + + - name: Test container security + uses: aquasecurity/trivy-action@master + with: + image-ref: 'ocp-tool:test-${{ matrix.target }}' + format: 'table' + severity: 'CRITICAL,HIGH' + continue-on-error: true + + build-singularity: + name: Build Singularity Image + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Singularity + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.11.4 + + - name: Build Singularity container + run: | + sudo singularity build ocp-tool.sif workflow/containers/singularity.def + + - name: Test Singularity container + run: | + # Test basic functionality + singularity exec ocp-tool.sif python --version + singularity exec ocp-tool.sif snakemake --version + + # Test OCP-tool imports + singularity exec ocp-tool.sif python -c "import ocp_tool; print('✓ OCP-tool import successful')" + + - name: Upload Singularity image as artifact + uses: actions/upload-artifact@v4 + with: + name: singularity-image + path: ocp-tool.sif + retention-days: 30 + + container-scan: + name: Container Security Scan + runs-on: ubuntu-latest + needs: build-docker + if: github.event_name != 'pull_request' + + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-production + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' + + integration-test: + name: Integration Test with Containers + runs-on: ubuntu-latest + needs: build-docker + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test image + uses: docker/build-push-action@v5 + with: + context: . + file: workflow/containers/Dockerfile + target: production + load: true + tags: ocp-tool:integration-test + cache-from: type=gha + + - name: Create test data structure + run: | + mkdir -p test_data/{input,output} + mkdir -p test_data/input/{gaussian_grids_full,gaussian_grids_linear_reduced,openifs_input_default,runoff_map_default,fesom_mesh} + mkdir -p test_data/output/{openifs_input_modified,oasis_mct3_input,runoff_map_modified,plots} + + # Create minimal test config + cat > test_data/test_config.yaml << EOF + res_num: 159 + truncation_type: "linear" + exp_name_oifs: "test" + num_fields: 50 + grid_name_oce: "TEST" + cavity: false + interp_res: "r360x181" + fesom_grid_file_path: "/app/data/input/test_mesh.nc" + do_paleo: false + manual_basin_removal: [] + EOF + + - name: Run integration test + run: | + docker run --rm \ + -v $(pwd)/test_data:/app/data \ + -w /app \ + ocp-tool:integration-test \ + snakemake --dry-run --configfile /app/data/test_config.yaml + + echo "✓ Integration test completed successfully" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..38fdb87 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,200 @@ +name: CI + +on: + push: + branches: [ master, main, develop ] + pull_request: + branches: [ master, main ] + workflow_dispatch: + +jobs: + test-workflow: + name: Test Snakemake Workflow + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + environments: dev + + - name: Lint Snakefile + run: pixi run workflow-lint + + - name: Validate workflow configuration + run: | + pixi run -e dev python -c "import yaml; yaml.safe_load(open('workflow/config/config.yaml'))" + echo "✓ Configuration file is valid YAML" + + - name: Check Python scripts syntax + run: | + pixi run -e dev python -m py_compile ocp-tool.py + pixi run -e dev python -m py_compile ocp_tool/*.py + find workflow/scripts -name "*.py" -exec pixi run -e dev python -m py_compile {} + + echo "✓ All Python scripts have valid syntax" + + - name: Dry run Snakemake workflow + run: | + # Create minimal test data structure + mkdir -p input/{gaussian_grids_full,gaussian_grids_linear_reduced,openifs_input_default,runoff_map_default,fesom_mesh} + mkdir -p output/{openifs_input_modified,oasis_mct3_input,runoff_map_modified,plots} + + # Test dry run with modified config for CI + cat > workflow/test_config.yaml << EOF + res_num: 159 + truncation_type: "linear" + exp_name_oifs: "test" + num_fields: 50 + grid_name_oce: "TEST" + cavity: false + interp_res: "r360x181" + fesom_grid_file_path: "/tmp/test_mesh.nc" + do_paleo: false + manual_basin_removal: [] + EOF + + pixi run workflow-dry --configfile test_config.yaml + echo "✓ Snakemake workflow dry run successful" + + - name: Test import of ocp_tool modules + run: | + pixi run -e dev python -c " + import sys + sys.path.insert(0, '.') + from ocp_tool import ocp_tool + print('✓ Successfully imported ocp_tool module') + " + + code-quality: + name: Code Quality Checks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + environments: dev + + - name: Run flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + pixi run -e dev flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.git,__pycache__,build,dist + # Exit-zero treats all errors as warnings + pixi run -e dev flake8 . --count --exit-zero --max-complexity=15 --max-line-length=120 --statistics --exclude=.git,__pycache__,build,dist + + - name: Check code formatting with black + run: | + pixi run -e dev black --check --diff --exclude=.git . || echo "::warning::Code formatting issues found. Run 'pixi run format' to fix." + + - name: Check import sorting with isort + run: | + pixi run -e dev isort --check-only --diff . || echo "::warning::Import sorting issues found. Run 'pixi run sort-imports' to fix." + + notebook-tests: + name: Test Jupyter Notebooks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + environments: jupyter + + - name: Validate notebook structure + run: | + pixi run -e jupyter pip install nbformat + pixi run -e jupyter python -c " + import nbformat + import json + + nb = nbformat.read('ocp_tool.ipynb', as_version=4) + print(f'✓ Notebook has {len(nb.cells)} cells') + print(f'✓ Notebook format version: {nb.nbformat}.{nb.nbformat_minor}') + + # Check for basic structure + code_cells = [c for c in nb.cells if c.cell_type == 'code'] + print(f'✓ Found {len(code_cells)} code cells') + " + + - name: Check notebook for output clearing + run: | + pixi run -e jupyter python -c " + import nbformat + import sys + + nb = nbformat.read('ocp_tool.ipynb', as_version=4) + has_outputs = False + + for cell in nb.cells: + if cell.cell_type == 'code' and cell.outputs: + has_outputs = True + break + + if has_outputs: + print('::warning::Notebook contains outputs. Consider clearing before commit.') + else: + print('✓ Notebook outputs are cleared') + " + + documentation: + name: Documentation Check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check README files + run: | + for readme in README.md workflow/README.md; do + if [ -f "$readme" ]; then + echo "✓ Found $readme" + else + echo "::error::Missing $readme" + exit 1 + fi + done + + - name: Check for broken links in documentation + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + config-file: '.github/markdown-link-check-config.json' + continue-on-error: true + + security-scan: + name: Security Scanning + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy security scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + continue-on-error: true + + - name: Upload Trivy results to GitHub Security + uses: github/codeql-action/upload-sarif@v2 + if: always() + with: + sarif_file: 'trivy-results.sarif' + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..f9f1303 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,308 @@ +name: Documentation + +on: + push: + branches: [ master, main ] + paths: + - 'docs/**' + - '**.md' + - '**.py' + - 'workflow/**' + pull_request: + branches: [ master, main ] + paths: + - 'docs/**' + - '**.md' + - '**.py' + - 'workflow/**' + workflow_dispatch: + +jobs: + build-docs: + name: Build Documentation + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for proper versioning + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + environments: docs + + - name: Generate API documentation + run: | + # Create docs directory structure if it doesn't exist + mkdir -p docs/{source,build} + + # Generate Sphinx configuration if it doesn't exist + if [ ! -f docs/source/conf.py ]; then + cat > docs/source/conf.py << 'EOF' + import os + import sys + sys.path.insert(0, os.path.abspath('../..')) + + project = 'OCP-tool' + copyright = '2024, OCP-tool developers' + author = 'OCP-tool developers' + + extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinx.ext.intersphinx', + 'nbsphinx', + 'myst_parser', + ] + + templates_path = ['_templates'] + exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + html_theme = 'sphinx_rtd_theme' + html_static_path = ['_static'] + + # Notebook execution settings + nbsphinx_execute = 'never' + nbsphinx_allow_errors = True + + # Intersphinx mapping + intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'numpy': ('https://numpy.org/doc/stable/', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), + } + + # MyST settings + myst_enable_extensions = [ + "colon_fence", + "deflist", + "html_image", + "linkify", + "replacements", + "smartquotes", + "tasklist", + ] + EOF + fi + + # Generate main index if it doesn't exist + if [ ! -f docs/source/index.rst ]; then + cat > docs/source/index.rst << 'EOF' + OCP-tool Documentation + ====================== + + OpenIFS Coupling Preparation Tool for climate modeling. + + .. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + workflow + api + notebooks + examples + + Indices and tables + ================== + + * :ref:`genindex` + * :ref:`modindex` + * :ref:`search` + EOF + fi + + # Build API docs with autoapi + pixi run -e docs sphinx-apidoc -f -o docs/source/api ocp_tool + + - name: Convert notebooks for documentation + run: | + mkdir -p docs/source/notebooks + + # Copy main notebook + if [ -f ocp_tool.ipynb ]; then + cp ocp_tool.ipynb docs/source/notebooks/ + fi + + # Create notebook index + cat > docs/source/notebooks.rst << 'EOF' + Jupyter Notebooks + ================= + + Example notebooks demonstrating OCP-tool usage. + + .. toctree:: + :maxdepth: 2 + + notebooks/ocp_tool + EOF + + - name: Create additional documentation pages + run: | + # Installation guide + cat > docs/source/installation.md << 'EOF' + # Installation + + ## Requirements + + - Python 3.8+ + - Conda/Mamba package manager + - NetCDF libraries + - ECCODES for GRIB processing + + ## Quick Install + + ```bash + git clone https://github.com/pgierz/ocp-tool.git + cd ocp-tool + conda env create -f environment.yaml + conda activate ocp-tool + pip install -e . + ``` + + ## Container Installation + + ### Docker + ```bash + docker pull ghcr.io/pgierz/ocp-tool:latest-production + ``` + + ### Singularity + ```bash + singularity pull ocp-tool.sif oras://ghcr.io/pgierz/ocp-tool:latest-singularity + ``` + EOF + + # Workflow documentation + cat > docs/source/workflow.md << 'EOF' + # Snakemake Workflow + + The OCP-tool provides a complete Snakemake workflow for automated processing. + + ## Configuration + + Edit `workflow/config/config.yaml`: + + ```yaml + res_num: 159 + truncation_type: "linear" + exp_name_oifs: "abda" + grid_name_oce: "CORE2" + ``` + + ## Execution + + ```bash + # Local execution + snakemake --cores 4 --use-conda + + # HPC cluster + snakemake --profile workflow/profiles/slurm + + # Container execution + docker run --rm -v $(pwd):/app/data ocp-tool:latest snakemake --cores 4 + ``` + + ## Workflow Rules + + - `prepare_gaussian_grids`: Process OpenIFS grid files + - `process_fesom_grid`: Handle ocean model grids + - `modify_land_sea_mask`: Core LSM modification + - `generate_oasis_files`: Create OASIS3-MCT files + - `modify_runoff_maps`: Adjust runoff routing + - `generate_plots`: Create visualizations + EOF + + # Examples page + cat > docs/source/examples.md << 'EOF' + # Examples + + ## Basic Usage + + ```python + from ocp_tool import ocp_tool + + # Generate coordinate and area fields + center_lats, center_lons, crn_lats, crn_lons, gridcell_area, lons_list, NN = \ + ocp_tool.generate_coord_area( + res_num=159, + input_path_reduced_grid="input/gaussian_grids_linear_reduced/", + input_path_full_grid="input/gaussian_grids_full/", + truncation_type="linear" + ) + ``` + + ## Workflow Examples + + See the `workflow/` directory for complete Snakemake workflow examples. + EOF + + - name: Build Sphinx documentation + run: | + pixi run docs-build + + - name: Upload documentation artifact + uses: actions/upload-artifact@v4 + with: + name: documentation + path: docs/build/html/ + retention-days: 30 + + deploy-docs: + name: Deploy Documentation + runs-on: ubuntu-latest + needs: build-docs + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' + + permissions: + contents: read + pages: write + id-token: write + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Download documentation artifact + uses: actions/download-artifact@v4 + with: + name: documentation + path: ./docs + + - name: Setup Pages + uses: actions/configure-pages@v4 + + - name: Upload to GitHub Pages + uses: actions/upload-pages-artifact@v3 + with: + path: ./docs + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + link-check: + name: Check Documentation Links + runs-on: ubuntu-latest + needs: build-docs + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download documentation artifact + uses: actions/download-artifact@v4 + with: + name: documentation + path: ./docs + + - name: Check HTML links + uses: lycheeverse/lychee-action@v1 + with: + args: --verbose --no-progress --exclude-all-private docs/**/*.html + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c0d24a6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,211 @@ +name: Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + tag: + description: 'Release tag' + required: true + type: string + +jobs: + create-release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + version: ${{ steps.get_version.outputs.version }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version + id: get_version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.tag }}" + else + VERSION=${GITHUB_REF#refs/tags/} + fi + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + - name: Generate changelog + id: changelog + run: | + # Generate changelog from git commits since last tag + LAST_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -z "$LAST_TAG" ]; then + CHANGES=$(git log --pretty=format:"- %s (%h)" --no-merges) + else + CHANGES=$(git log ${LAST_TAG}..HEAD --pretty=format:"- %s (%h)" --no-merges) + fi + + cat > CHANGELOG.md << EOF + # Changes in ${{ steps.get_version.outputs.version }} + + ${CHANGES} + + ## Full Changelog + + See [commits](https://github.com/${{ github.repository }}/compare/${LAST_TAG}...${{ steps.get_version.outputs.version }}) for full details. + EOF + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.get_version.outputs.version }} + release_name: Release ${{ steps.get_version.outputs.version }} + body_path: CHANGELOG.md + draft: false + prerelease: ${{ contains(steps.get_version.outputs.version, '-') }} + + build-release-containers: + name: Build Release Containers + needs: create-release + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + target: [production, development] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: | + suffix=-${{ matrix.target }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push release image + uses: docker/build-push-action@v5 + with: + context: . + file: workflow/containers/Dockerfile + target: ${{ matrix.target }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + build-singularity-release: + name: Build Singularity Release + needs: create-release + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Singularity + uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.11.4 + + - name: Build Singularity container + run: | + sudo singularity build ocp-tool-${{ needs.create-release.outputs.version }}.sif workflow/containers/singularity.def + + - name: Upload Singularity image to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./ocp-tool-${{ needs.create-release.outputs.version }}.sif + asset_name: ocp-tool-${{ needs.create-release.outputs.version }}.sif + asset_content_type: application/octet-stream + + upload-workflow-assets: + name: Upload Workflow Assets + needs: create-release + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create workflow archive + run: | + tar -czf ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz \ + workflow/ \ + environment.yaml \ + README.md \ + --exclude='workflow/logs' \ + --exclude='workflow/temp' + + - name: Upload workflow archive to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: ./ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz + asset_name: ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz + asset_content_type: application/gzip + + notify-release: + name: Notify Release + needs: [create-release, build-release-containers, build-singularity-release] + runs-on: ubuntu-latest + if: always() + + steps: + - name: Create release summary + run: | + cat > $GITHUB_STEP_SUMMARY << EOF + # 🚀 Release ${{ needs.create-release.outputs.version }} Complete + + ## 📦 Available Assets + + - **Docker Images**: \`ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production\` + - **Development Image**: \`ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-development\` + - **Singularity Container**: \`ocp-tool-${{ needs.create-release.outputs.version }}.sif\` + - **Workflow Archive**: \`ocp-tool-workflow-${{ needs.create-release.outputs.version }}.tar.gz\` + + ## 🐳 Quick Start with Docker + + \`\`\`bash + docker pull ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production + docker run --rm -v \$(pwd):/app/data ghcr.io/${{ github.repository }}:${{ needs.create-release.outputs.version }}-production snakemake --cores 4 + \`\`\` + + ## 📖 Documentation + + Visit the [documentation](https://${{ github.repository_owner }}.github.io/ocp-tool/) for usage guides and API reference. + EOF \ No newline at end of file diff --git a/ocp_tool/__init__.py b/ocp_tool/__init__.py index e69de29..ffb0900 100644 --- a/ocp_tool/__init__.py +++ b/ocp_tool/__init__.py @@ -0,0 +1,3 @@ +"""OCP-tool: OpenIFS Coupling Preparation Tool.""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..99541d3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,179 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "ocp-tool" +description = "Tool to generate OASIS files for coupling OpenIFS, FESOM2, and NEMO" +authors = [ + {name = "Jan Streffing", email = "jan.streffing@awi.de"} +] +readme = "README.md" +license = {file = "licence"} +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: Atmospheric Science", +] +dependencies = [ + "numpy", + "netcdf4", + "eccodes", + "matplotlib", + "pandas", + "pyyaml", + "tqdm", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/pgierz/ocp-tool" +Repository = "https://github.com/pgierz/ocp-tool" +Issues = "https://github.com/pgierz/ocp-tool/issues" + +[project.entry-points."scriptengine.tasks"] +"ocpt.main" = "ocp_tool.scriptengine_task:OCPTool" + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "black", + "isort", + "flake8", + "mypy", +] +jupyter = [ + "jupyterlab", + "notebook", + "ipykernel", +] +docs = [ + "sphinx", + "sphinx-rtd-theme", + "nbsphinx", + "myst-parser", + "sphinx-autoapi", +] + +[tool.setuptools] +packages = ["ocp_tool"] + +[tool.setuptools.dynamic] +version = {attr = "ocp_tool.__version__"} + +# Pixi configuration +[tool.pixi.project] +name = "ocp-tool" +description = "Tool to generate OASIS files for coupling OpenIFS, FESOM2, and NEMO" +authors = ["Jan Streffing "] +channels = ["conda-forge", "bioconda", "eumetsat"] +platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"] + +[tool.pixi.dependencies] +python = ">=3.8" +numpy = "*" +netcdf4 = "*" +matplotlib = "*" +basemap = "*" +pandas = "*" +pyyaml = "*" +python-eccodes = "*" +git = "*" +pip = "*" +tqdm = "*" +snakemake-minimal = ">=7.0" + +[tool.pixi.pypi-dependencies] +# Use modernized pyfesom2 branch with Python 3.11+ support +pyfesom2 = { git = "https://github.com/FESOM/pyfesom2.git", branch = "modernize-packaging" } + +[tool.pixi.feature.jupyter.dependencies] +jupyterlab = "*" +notebook = "*" +ipykernel = "*" + +[tool.pixi.feature.dev.dependencies] +pytest = "*" +pytest-cov = "*" +black = "*" +isort = "*" +flake8 = "*" +mypy = "*" + +[tool.pixi.feature.docs.dependencies] +sphinx = "*" +sphinx-rtd-theme = "*" +nbsphinx = "*" +myst-parser = "*" +sphinx-autoapi = "*" + +[tool.pixi.environments] +default = { solve-group = "default" } +jupyter = { features = ["jupyter"], solve-group = "default" } +dev = { features = ["dev"], solve-group = "default" } +docs = { features = ["docs"], solve-group = "default" } +full = { features = ["jupyter", "dev", "docs"], solve-group = "default" } + +[tool.pixi.tasks] +lint = "flake8 ocp_tool" +format = "black ocp_tool" +sort-imports = "isort ocp_tool" +type-check = "mypy ocp_tool" +test = "pytest" +clean-format = { depends-on = ["format", "sort-imports"] } +jupyter = "jupyter lab" +notebook = "jupyter notebook" +docs-build = "sphinx-build -W -b html docs/source docs/build/html" +docs-clean = "rm -rf docs/build" +docs-serve = "python -m http.server 8000 --directory docs/build/html" +workflow-dry = { cmd = "snakemake --dry-run", cwd = "workflow" } +workflow-lint = { cmd = "snakemake --lint", cwd = "workflow" } +workflow-run = { cmd = "snakemake --cores 4 --use-conda", cwd = "workflow" } +build = "python -m build" +install-dev = "pip install -e ." + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["ocp_tool"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--cov=ocp_tool", + "--cov-report=term-missing", + "--cov-report=html", +] \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 7fd3247..0000000 --- a/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -import setuptools - - -setuptools.setup( - name='ocp-tool', - author='Jan Streffing', - author_email='jan.streffing@awi.de', - description='Tool to generate OASIS files for coupling OpenIFS, FESOM2, and NEMO', - url='https://github.com/JanStreffing/ocp-tool', - packages=setuptools.find_packages(), - python_requires='>=3.6', - install_requires=[ - 'numpy', - 'netcdf4', - 'eccodes', - 'pyfesom2', - ], - entry_points={ - 'scriptengine.tasks': [ - 'ocpt.main = ocp_tool.scriptengine_task:OCPTool', - ], - }, -) diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 0000000..988ceb7 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,35 @@ +# OCP-tool Snakemake Workflow + +This directory contains the Snakemake workflow for automated climate model preparation. + +## Usage + +Configure your settings in `config/config.yaml`, then run: + +```bash +# Local execution +snakemake --cores 4 + +# With pixi +pixi run workflow-run + +# Dry run to check workflow +pixi run workflow-dry +``` + +## Configuration + +Edit `config/config.yaml` to specify: +- Grid resolution (`res_num`) +- Experiment name (`exp_name_oifs`) +- Ocean grid name (`grid_name_oce`) +- Input/output paths + +## Rules + +- `prepare_gaussian_grids`: Process OpenIFS grid files +- `process_fesom_grid`: Handle ocean model grids +- `modify_land_sea_mask`: Core LSM modification +- `generate_oasis_files`: Create OASIS3-MCT files +- `modify_runoff_maps`: Adjust runoff routing +- `generate_plots`: Create visualizations \ No newline at end of file