Skip to content

feat(dynamodb)!: changed to dap registry get_pipelines to be by id and added get_pipelines_by_name#22

Merged
mehalter merged 1 commit into
mainfrom
20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id
May 15, 2026
Merged

feat(dynamodb)!: changed to dap registry get_pipelines to be by id and added get_pipelines_by_name#22
mehalter merged 1 commit into
mainfrom
20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id

Conversation

@mehalter
Copy link
Copy Markdown
Member

TO TEST

Ran this locally. Feel free to recreate:

>>> from capepy.aws.dynamodb import PipelineTable
>>> pt=PipelineTable(table_name="cape-dap-registry-DataAnalysisPipelineRegistry")
>>> import pprint
>>> from capepy.aws.dynamodb import PipelineTable
>>> 
>>> 
>>> pt.get_pipeline("bactopia-ont-v3.2.0")
{'version': 'v3.2.0', 'pipeline_type': 'nextflow', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'pipeline_id': 'bactopia-ont-v3.2.0', 'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}, 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}, 'project': 'bactopia/bactopia'}
>>> 
>>> 
>>> 
>>> items = pt.get_pipelines_by_name("Bactopia ONT Sample")
>>> len(items)
2
>>> items
[{'profile': {'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-dev'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-dev': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'required': ['--sample', '--ont', '--outdir'], 'properties': {'--outdir': {'title': 'S3 Location of the output directory', 'type': 'string'}, '--max_memory': {'title': 'Max Memory', 'type': 'string', 'default': '24.GB'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--ont': {'type': 'string', 'title': 'ONT'}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False}, 'pipelineName': 'Bactopia ONT Sample', 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with the development release", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'dev', 'inherits': ['bactopia-base-dev'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-dev'}, 'project': 'bactopia/bactopia', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'version': 'dev', 'pipeline_id': 'bactopia-ont-dev', 'pipeline_type': 'nextflow', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}}, {'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}, 'pipeline_id': 'bactopia-ont-v3.2.0', 'project': 'bactopia/bactopia', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'pipeline_type': 'nextflow', 'version': 'v3.2.0', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}}]
>>> 
>>> 
>>> 
>>> item = pt.get_pipeline_by_name_ver("Bactopia ONT Sample", "v3.2.0")
>>> len(item)
1
>>> item
[{'pipeline_runnable': True, 'version': 'v3.2.0', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}, 'pipeline_name': 'Bactopia ONT Sample', 'project': 'bactopia/bactopia', 'pipeline_type': 'nextflow', 'pipeline_id': 'bactopia-ont-v3.2.0', 'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}}]
>>> 
>>> 
>>> 
>>> bad_vals = pt.get_pipeline("awsad")
>>> bad_vals
>>> bad_vals == None
True
>>> 
>>> 
>>> 
>>> bad_vals = pt.get_pipeline_by_name("awsad")
>>> bad_vals
[]
>>> bad_vals = pt.get_pipeline_by_name_ver("Bactopia ONT Sample", "awsad")
>>> bad_vals
[]


… search the secondary index by name and optionally version
@mehalter
Copy link
Copy Markdown
Member Author

this is a remerge of #21

@mehalter mehalter merged commit 10e3cae into main May 15, 2026
21 checks passed
@mehalter mehalter deleted the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:38
@mehalter mehalter restored the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:39
@mehalter mehalter deleted the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:41
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants