AWI-HPC Work

.claude/settings.local.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git cherry-pick:*)",
+      "Bash(gh:*)",
+      "Bash(act:*)",
+      "Bash(pixi install:*)",
+      "Bash(git for-each-ref:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

.github/markdown-link-check-config.json

@@ -0,0 +1,31 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "^http://localhost"
+    },
+    {
+      "pattern": "^https://127.0.0.1"
+    },
+    {
+      "pattern": "^file://"
+    }
+  ],
+  "replacementPatterns": [
+    {
+      "pattern": "^/",
+      "replacement": "{{BASEURL}}/"
+    }
+  ],
+  "httpHeaders": [
+    {
+      "urls": ["https://github.com"],
+      "headers": {
+        "Accept": "text/html"
+      }
+    }
+  ],
+  "timeout": "20s",
+  "retryOn429": true,
+  "retryCount": 3,
+  "fallbackHttpStatus": [400, 401, 403, 404, 405, 500, 502, 503, 504]
+}

.github/workflows/build-containers.yml

@@ -0,0 +1,235 @@
+name: Build and Push Containers
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ 'v*' ]
+  pull_request:
+    branches: [ master, main ]
+  workflow_dispatch:
+    inputs:
+      push_images:
+        description: 'Push images to registry'
+        required: false
+        default: false
+        type: boolean
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-docker:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        target: [production, development]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-${{ matrix.target }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: workflow/containers/Dockerfile
+          target: ${{ matrix.target }}
+          push: ${{ github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') || github.event.inputs.push_images == 'true') }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64
+
+  test-containers:
+    name: Test Container Images  
+    runs-on: ubuntu-latest
+    needs: build-docker
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+
+    strategy:
+      matrix:
+        target: [production, development]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build test image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: workflow/containers/Dockerfile
+          target: ${{ matrix.target }}
+          load: true
+          tags: ocp-tool:test-${{ matrix.target }}
+          cache-from: type=gha
+
+      - name: Test production container
+        if: matrix.target == 'production'
+        run: |
+          # Test basic functionality
+          docker run --rm ocp-tool:test-production python --version
+          docker run --rm ocp-tool:test-production snakemake --version
+
+          # Test OCP-tool imports
+          docker run --rm ocp-tool:test-production python -c "import ocp_tool; print('✓ OCP-tool import successful')"
+
+          # Test conda environment
+          docker run --rm ocp-tool:test-production conda list | grep -E "(numpy|matplotlib|netcdf4)"
+
+      - name: Test development container
+        if: matrix.target == 'development'
+        run: |
+          # Test Jupyter installation
+          docker run --rm ocp-tool:test-development jupyter --version
+          docker run --rm ocp-tool:test-development jupyter kernelspec list
+
+      - name: Test container security
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'ocp-tool:test-${{ matrix.target }}'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+        continue-on-error: true
+
+  build-singularity:
+    name: Build Singularity Image
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Singularity
+        uses: eWaterCycle/setup-singularity@v7
+        with:
+          singularity-version: 3.11.4
+
+      - name: Build Singularity container
+        run: |
+          sudo singularity build ocp-tool.sif workflow/containers/singularity.def
+
+      - name: Test Singularity container
+        run: |
+          # Test basic functionality
+          singularity exec ocp-tool.sif python --version
+          singularity exec ocp-tool.sif snakemake --version
+
+          # Test OCP-tool imports  
+          singularity exec ocp-tool.sif python -c "import ocp_tool; print('✓ OCP-tool import successful')"
+
+      - name: Upload Singularity image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: singularity-image
+          path: ocp-tool.sif
+          retention-days: 30
+
+  container-scan:
+    name: Container Security Scan
+    runs-on: ubuntu-latest
+    needs: build-docker
+    if: github.event_name != 'pull_request'
+
+    steps:
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-production
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy results to GitHub Security
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  integration-test:
+    name: Integration Test with Containers
+    runs-on: ubuntu-latest
+    needs: build-docker
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build test image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: workflow/containers/Dockerfile
+          target: production
+          load: true
+          tags: ocp-tool:integration-test
+          cache-from: type=gha
+
+      - name: Create test data structure
+        run: |
+          mkdir -p test_data/{input,output}
+          mkdir -p test_data/input/{gaussian_grids_full,gaussian_grids_linear_reduced,openifs_input_default,runoff_map_default,fesom_mesh}
+          mkdir -p test_data/output/{openifs_input_modified,oasis_mct3_input,runoff_map_modified,plots}
+
+          # Create minimal test config
+          cat > test_data/test_config.yaml << EOF
+          res_num: 159
+          truncation_type: "linear"
+          exp_name_oifs: "test"
+          num_fields: 50
+          grid_name_oce: "TEST"
+          cavity: false
+          interp_res: "r360x181"
+          fesom_grid_file_path: "/app/data/input/test_mesh.nc"
+          do_paleo: false
+          manual_basin_removal: []
+          EOF
+
+      - name: Run integration test
+        run: |
+          docker run --rm \
+            -v $(pwd)/test_data:/app/data \
+            -w /app \
+            ocp-tool:integration-test \
+            snakemake --dry-run --configfile /app/data/test_config.yaml
+
+          echo "✓ Integration test completed successfully"