Skip to content

feat(dynamodb)!: changed to dap registry get_pipelines to be by id and added get_pipelines_by_name - #22

Merged
mehalter merged 1 commit into
mainfrom
20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id
May 15, 2026
Merged

mehalter merged 1 commit into
mainfrom
20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id

Conversation

@mehalter

Copy link
Copy Markdown
Member

TO TEST

Ran this locally. Feel free to recreate:

>>> from capepy.aws.dynamodb import PipelineTable
>>> pt=PipelineTable(table_name="cape-dap-registry-DataAnalysisPipelineRegistry")
>>> import pprint
>>> from capepy.aws.dynamodb import PipelineTable
>>> 
>>> 
>>> pt.get_pipeline("bactopia-ont-v3.2.0")
{'version': 'v3.2.0', 'pipeline_type': 'nextflow', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'pipeline_id': 'bactopia-ont-v3.2.0', 'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}, 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}, 'project': 'bactopia/bactopia'}
>>> 
>>> 
>>> 
>>> items = pt.get_pipelines_by_name("Bactopia ONT Sample")
>>> len(items)
2
>>> items
[{'profile': {'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-dev'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-dev': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'required': ['--sample', '--ont', '--outdir'], 'properties': {'--outdir': {'title': 'S3 Location of the output directory', 'type': 'string'}, '--max_memory': {'title': 'Max Memory', 'type': 'string', 'default': '24.GB'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--ont': {'type': 'string', 'title': 'ONT'}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False}, 'pipelineName': 'Bactopia ONT Sample', 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with the development release", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'dev', 'inherits': ['bactopia-base-dev'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-dev'}, 'project': 'bactopia/bactopia', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'version': 'dev', 'pipeline_id': 'bactopia-ont-dev', 'pipeline_type': 'nextflow', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}}, {'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}, 'pipeline_id': 'bactopia-ont-v3.2.0', 'project': 'bactopia/bactopia', 'pipeline_name': 'Bactopia ONT Sample', 'pipeline_runnable': True, 'pipeline_type': 'nextflow', 'version': 'v3.2.0', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}}]
>>> 
>>> 
>>> 
>>> item = pt.get_pipeline_by_name_ver("Bactopia ONT Sample", "v3.2.0")
>>> len(item)
1
>>> item
[{'pipeline_runnable': True, 'version': 'v3.2.0', 'nextflow_config': {'aws': {'batch': {'cliPath': '/usr/bin/aws', 'delayBetweenAttempts': '5 sec', 'maxTransferAttempts': Decimal('3')}, 'client': {'uploadStorageClass': 'INTELLIGENT_TIERING', 'connectionTimeout': Decimal('10000'), 'storageEncryption': 'AES256', 'maxConnections': Decimal('20')}, 'secretKey': '<YOUR S3 SECRET KEY>', 'region': 'us-east-2', 'accessKey': '<YOUR S3 ACCESS KEY>'}}, 'pipeline_name': 'Bactopia ONT Sample', 'project': 'bactopia/bactopia', 'pipeline_type': 'nextflow', 'pipeline_id': 'bactopia-ont-v3.2.0', 'profile': {'pipelineName': 'Bactopia ONT Sample', 'parametersSchema': {'allOf': [{'$ref': '#/$defs/bactopia-base-3.2.0'}], '$schema': 'https://json-schema.org/draft/2020-12/schema', '$defs': {'bactopia-base-3.2.0': {'$defs': {}, '$schema': 'https://json-schema.org/draft/2020-12/schema', 'type': 'object', 'properties': {'--aws_volumes': {'const': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro', 'default': '/opt/conda:/mnt/conda,/mnt/nextflow_shared_data:/mnt/nextflow_shared_data:ro'}, '-profile': {'const': 'aws', 'default': 'aws'}}}}, 'type': 'object', 'properties': {'--outdir': {'type': 'string', 'title': 'S3 Location of the output directory'}, '--max_memory': {'type': 'string', 'title': 'Max Memory', 'default': '24.GB'}, '--ont': {'type': 'string', 'title': 'ONT'}, '--max_cpus': {'default': Decimal('8'), 'type': 'integer', 'title': 'Max CPUs', 'minimum': Decimal('1')}, '--sample': {'type': 'string', 'title': 'Sample Name'}}, 'unevaluatedProperties': False, 'required': ['--sample', '--ont', '--outdir']}, 'pipelineDescription': "Execute Bactopia's ONT sample sequencing workflow with v3.2.0", 'uiSchema': {'type': 'VerticalLayout', 'elements': [{'type': 'Control', 'scope': '#/properties/--max_cpus'}, {'type': 'Control', 'scope': '#/properties/--max_memory'}, {'type': 'Control', 'scope': '#/properties/--ont'}, {'type': 'Control', 'scope': '#/properties/--sample'}]}, 'project': 'bactopia/bactopia', 'submission': {'encoding': 'cli-string', 'optionsFieldName': 'nextflowOptions'}, 'pipelineType': 'nextflow', 'version': 'v3.2.0', 'inherits': ['bactopia-base-3.2.0'], 'pipelineRunnable': True, 'pipelineId': 'bactopia-ont-v3.2.0'}}]
>>> 
>>> 
>>> 
>>> bad_vals = pt.get_pipeline("awsad")
>>> bad_vals
>>> bad_vals == None
True
>>> 
>>> 
>>> 
>>> bad_vals = pt.get_pipeline_by_name("awsad")
>>> bad_vals
[]
>>> bad_vals = pt.get_pipeline_by_name_ver("Bactopia ONT Sample", "awsad")
>>> bad_vals
[]


… search the secondary index by name and optionally version
@mehalter

Copy link
Copy Markdown
Member Author

this is a remerge of #21

@mehalter
mehalter merged commit 10e3cae into main May 15, 2026
21 checks passed
@mehalter
mehalter deleted the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:38
@mehalter
mehalter restored the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:39
@mehalter
mehalter deleted the 20-rework-dap-registry-query-for-new-gsi-and-make-primary-query-get_pipeline-by-id branch May 15, 2026 14:41
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants