From 0bf9a9d3729da1e6e26324d6e069600f7d30d07f Mon Sep 17 00:00:00 2001
From: Micah Halter <micah.halter@gtri.gatech.edu>
Date: Wed, 6 May 2026 15:23:58 -0400
Subject: [PATCH] feat(pipelines): mount shared nextflow data to AWS batch jobs

---
 Pulumi.cape-cod-dev.yaml                                  | 8 ++++++++
 .../analysis-pipelines/bactopia/bactopia-base-3.2.0.json  | 4 ++++
 assets/analysis-pipelines/bactopia/bactopia-base-dev.json | 4 ++++
 .../bactopia/kraken2-bactopia-3.2.0.json                  | 5 +++++
 .../analysis-pipelines/bactopia/kraken2-bactopia-dev.json | 5 +++++
 assets/containers/nextflow-kickstart/entrypoint.sh        | 4 ++++
 capeinfra/pipeline/batch.py                               | 6 ++++++
 7 files changed, 36 insertions(+)

diff --git a/Pulumi.cape-cod-dev.yaml b/Pulumi.cape-cod-dev.yaml
index 5a33bfc..b9f2ee8 100644
--- a/Pulumi.cape-cod-dev.yaml
+++ b/Pulumi.cape-cod-dev.yaml
@@ -1517,6 +1517,14 @@ config:
                               value: "1"
                             - type: MEMORY
                               value: "2048"
+                        mountPoints:
+                            - sourceVolume: "nextflow_shared_data"
+                              containerPath: "/mnt/nextflow_shared_data"
+                              readOnly: true
+                        volumes:
+                            - name: "nextflow_shared_data"
+                              efsVolumeConfiguration:
+                                  fileSystemId: fs-0cc7e588b5015c314
     # `cape-cod:datalakehouse` (mapping, required)
     # Contains configuration specific to the data lake house (DLH). The DLH
     # contains tributaries, which are compionents that consist of a pair of raw
diff --git a/assets/analysis-pipelines/bactopia/bactopia-base-3.2.0.json b/assets/analysis-pipelines/bactopia/bactopia-base-3.2.0.json
index 1bf3cdd..b3fe260 100644
--- a/assets/analysis-pipelines/bactopia/bactopia-base-3.2.0.json
+++ b/assets/analysis-pipelines/bactopia/bactopia-base-3.2.0.json
@@ -12,6 +12,10 @@
             "-profile": {
                 "const": "aws",
                 "default": "aws"
+            },
+            "--aws_volumes": {
+                "const": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro",
+                "default": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro"
             }
         }
     },
diff --git a/assets/analysis-pipelines/bactopia/bactopia-base-dev.json b/assets/analysis-pipelines/bactopia/bactopia-base-dev.json
index 3777c80..4b52ef6 100644
--- a/assets/analysis-pipelines/bactopia/bactopia-base-dev.json
+++ b/assets/analysis-pipelines/bactopia/bactopia-base-dev.json
@@ -12,6 +12,10 @@
             "-profile": {
                 "const": "aws",
                 "default": "aws"
+            },
+            "--aws_volumes": {
+                "const": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro",
+                "default": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro"
             }
         }
     },
diff --git a/assets/analysis-pipelines/bactopia/kraken2-bactopia-3.2.0.json b/assets/analysis-pipelines/bactopia/kraken2-bactopia-3.2.0.json
index 23bf167..563db07 100644
--- a/assets/analysis-pipelines/bactopia/kraken2-bactopia-3.2.0.json
+++ b/assets/analysis-pipelines/bactopia/kraken2-bactopia-3.2.0.json
@@ -15,6 +15,11 @@
                 "const": "kraken2",
                 "default": "kraken2"
             },
+            "--kraken2_db": {
+                "type": "string",
+                "title": "Kraken2 Database Path",
+                "default": "/mnt/nextflow_shared_data/kraken2"
+            },
             "--max_cpus": {
                 "type": "integer",
                 "title": "Max CPUs",
diff --git a/assets/analysis-pipelines/bactopia/kraken2-bactopia-dev.json b/assets/analysis-pipelines/bactopia/kraken2-bactopia-dev.json
index 10dd8c0..885c029 100644
--- a/assets/analysis-pipelines/bactopia/kraken2-bactopia-dev.json
+++ b/assets/analysis-pipelines/bactopia/kraken2-bactopia-dev.json
@@ -15,6 +15,11 @@
                 "const": "kraken2",
                 "default": "kraken2"
             },
+            "--kraken2_db": {
+                "type": "string",
+                "title": "Kraken2 Database Path",
+                "default": "/mnt/nextflow_shared_data/kraken2"
+            },
             "--max_cpus": {
                 "type": "integer",
                 "title": "Max CPUs",
diff --git a/assets/containers/nextflow-kickstart/entrypoint.sh b/assets/containers/nextflow-kickstart/entrypoint.sh
index 1f6cf8a..190df90 100644
--- a/assets/containers/nextflow-kickstart/entrypoint.sh
+++ b/assets/containers/nextflow-kickstart/entrypoint.sh
@@ -46,6 +46,10 @@ cat >/nextflow.config <<EOF
 process {
     executor = 'awsbatch'
     queue = '${PIPELINE_QUEUE}'
+    // symlink kraken data - THIS SHOULD BE MOVED TO PIPELINE SPECIFIC NEXTFLOW CONFIG
+    withName: '.*:KRAKEN2|KRAKEN2' {
+        stageInMode = 'symlink'
+    }
 }
 aws {
     region = '${AWS_REGION}'
diff --git a/capeinfra/pipeline/batch.py b/capeinfra/pipeline/batch.py
index dd64e7c..d46e4f8 100644
--- a/capeinfra/pipeline/batch.py
+++ b/capeinfra/pipeline/batch.py
@@ -144,6 +144,12 @@ def __init__(
             policy_arn="arn:aws:iam::aws:policy/AWSBatchFullAccess",
             opts=ResourceOptions(parent=self),
         )
+        aws.iam.RolePolicyAttachment(
+            f"{name}-instnc-efssvcroleatch",
+            role=self.instance_role.name,
+            policy_arn="arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess",
+            opts=ResourceOptions(parent=self),
+        )
         self.instance_role_profile = get_instance_profile(
             f"{name}-instnc-rl",
             self.instance_role,