feat(pipelines): mount shared nextflow data to AWS batch jobs

Pulumi.cape-cod-dev.yaml

@@ -1517,6 +1517,14 @@ config:
                               value: "1"
                             - type: MEMORY
                               value: "2048"
+                        mountPoints:
+                            - sourceVolume: "nextflow_shared_data"
+                              containerPath: "/mnt/nextflow_shared_data"
+                              readOnly: true
+                        volumes:
+                            - name: "nextflow_shared_data"
+                              efsVolumeConfiguration:
+                                  fileSystemId: fs-0cc7e588b5015c314
     # `cape-cod:datalakehouse` (mapping, required)
     # Contains configuration specific to the data lake house (DLH). The DLH
     # contains tributaries, which are compionents that consist of a pair of raw

assets/analysis-pipelines/bactopia/bactopia-base-3.2.0.json

@@ -12,6 +12,10 @@
             "-profile": {
                 "const": "aws",
                 "default": "aws"
+            },
+            "--aws_volumes": {
+                "const": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro",
+                "default": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro"
             }
         }
     },

assets/analysis-pipelines/bactopia/bactopia-base-dev.json

@@ -12,6 +12,10 @@
             "-profile": {
                 "const": "aws",
                 "default": "aws"
+            },
+            "--aws_volumes": {
+                "const": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro",
+                "default": "/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro"
             }
         }
     },

assets/analysis-pipelines/bactopia/kraken2-bactopia-3.2.0.json

@@ -15,6 +15,11 @@
                 "const": "kraken2",
                 "default": "kraken2"
             },
+            "--kraken2_db": {
+                "type": "string",
+                "title": "Kraken2 Database Path",
+                "default": "/mnt/nextflow_shared_data/kraken2"
+            },
             "--max_cpus": {
                 "type": "integer",
                 "title": "Max CPUs",

assets/analysis-pipelines/bactopia/kraken2-bactopia-dev.json

@@ -15,6 +15,11 @@
                 "const": "kraken2",
                 "default": "kraken2"
             },
+            "--kraken2_db": {
+                "type": "string",
+                "title": "Kraken2 Database Path",
+                "default": "/mnt/nextflow_shared_data/kraken2"
+            },
             "--max_cpus": {
                 "type": "integer",
                 "title": "Max CPUs",

assets/containers/nextflow-kickstart/entrypoint.sh

@@ -46,6 +46,10 @@ cat >/nextflow.config <<EOF
 process {
     executor = 'awsbatch'
     queue = '${PIPELINE_QUEUE}'
+    // symlink kraken data - THIS SHOULD BE MOVED TO PIPELINE SPECIFIC NEXTFLOW CONFIG
+    withName: '.*:KRAKEN2|KRAKEN2' {
+        stageInMode = 'symlink'
+    }
 }
 aws {
     region = '${AWS_REGION}'

capeinfra/pipeline/batch.py

@@ -144,6 +144,12 @@ def __init__(
             policy_arn="arn:aws:iam::aws:policy/AWSBatchFullAccess",
             opts=ResourceOptions(parent=self),
         )
+        aws.iam.RolePolicyAttachment(
+            f"{name}-instnc-efssvcroleatch",
+            role=self.instance_role.name,
+            policy_arn="arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess",
+            opts=ResourceOptions(parent=self),
+        )
         self.instance_role_profile = get_instance_profile(
             f"{name}-instnc-rl",
             self.instance_role,