feat!(config, website): flip config and website to segment-reference structure

backend/docs/organismWithSuborganisms.md

@@ -1,6 +1,6 @@
-# Solution Design - Organisms With Suborganisms
+# Solution Design - Organisms With Multiple References ("Subtypes")
 
-The purpose of this feature is to allow a single top level organism to contain multiple suborganisms.
+The purpose of this feature is to allow a single top level organism to contain multiple references, or multiple "suborganisms".
 
 Motivation:
 
@@ -39,7 +39,7 @@ defaultOrganisms:
       metadataAdd:
         - name: clade_cv_a10
           # tells the website to only show this field on the search page when CV-A10 is selected
-          onlyForSuborganism: CV-A10 
+          onlyForReference: CV-A10 
           preprocessing:
             args:
               segment: CV-A10
@@ -62,10 +62,10 @@ defaultOrganisms:
           perSegment: true
       website:
         <<: *website
-        # When the website needs to know which suborganism a sequence entry belongs to,
-        # it will look up the value of this metadata field.
+        # When the website needs to know which suborganism a sequence entry belongs to (i.e. which reference it aligns to),
+        # it will look up the value of this metadata field (this metadata field will exist for each segment, e.g. genotype_L, genotype_M).
         # Preprocessing must make sure that this field is always populated.
-        suborganismIdentifierField: genotype
+        referenceIdentifierField: genotype
     preprocessing:
       - <<: *preprocessing
         configFile:
@@ -88,22 +88,20 @@ defaultOrganisms:
     # `referenceGenomes` is now an object { suborganismName: referenceGenomeOfThatSuborganism }
     # The special suborganism name `singleReference` must be used when there is only a single suborganism
     referenceGenomes:
-      CV-A10:
-        nucleotideSequences:
-          - name: main
+      - name: main
+        references:
+          - reference_name: CV-A10
             sequence: "..."
             insdcAccessionFull: ...
-        genes: 
-          - name: VP4
-            sequence: "..."
-      EV-A71:
-        nucleotideSequences:
-          - name: main
+            genes: 
+              - name: VP4
+                sequence: "..."
+          - reference_name: EV-A71
             sequence: "..."
             insdcAccessionFull: ...
-        genes:
-          - name: VP2
-            sequence: "..."
+            genes:
+              - name: VP2
+                sequence: "..."
 ```
 
 The website will then receive the `referenceGenomes` as configured above.
@@ -322,10 +320,10 @@ The reference genome will be a product "suborganism x segment":
     {"name": "suborganism2", "sequence": "..."}
   ],
   "genes": [
-    {"name": "suborganism1_gene1", "sequence": "..."},
-    {"name": "suborganism1_gene2", "sequence": "..."},
-    {"name": "suborganism2_gene1", "sequence": "..."},
-    {"name": "suborganism2_gene2", "sequence": "..."}
+    {"name": "gene1_suborganism1", "sequence": "..."},
+    {"name": "gene2_suborganism1", "sequence": "..."},
+    {"name": "gene1_suborganism2", "sequence": "..."},
+    {"name": "gene2_suborganism2", "sequence": "..."}
   ]
 }
 ```
@@ -335,16 +333,16 @@ for multi-segment:
 ```json
 {
   "nucleotideSequences": [
-    {"name": "suborganism1_segment1", "sequence": "..."},
-    {"name": "suborganism1_segment2", "sequence": "..."},
-    {"name": "suborganism2_segment1", "sequence": "..."},
-    {"name": "suborganism2_segment2", "sequence": "..."}
+    {"name": "segment1_suborganism1", "sequence": "..."},
+    {"name": "segment2_suborganism1", "sequence": "..."},
+    {"name": "segment1_suborganism2", "sequence": "..."},
+    {"name": "segment2_suborganism2", "sequence": "..."}
   ],
   "genes": [
-    {"name": "suborganism1_gene1", "sequence": "..."},
-    {"name": "suborganism1_gene2", "sequence": "..."},
-    {"name": "suborganism2_gene1", "sequence": "..."},
-    {"name": "suborganism2_gene2", "sequence": "..."}
+    {"name": "gene1_suborganism1", "sequence": "..."},
+    {"name": "gene2_suborganism1", "sequence": "..."},
+    {"name": "gene1_suborganism2", "sequence": "..."},
+    {"name": "gene2_suborganism2", "sequence": "..."}
   ]
 }
 ```

ena-submission/scripts/get_ena_submission_list.py

@@ -43,24 +43,24 @@ class SubmissionResults:
 @dataclass
 class ENAOrganism:
     enaOrganismName: str  # noqa: N815
-    suborganismIdentifierField: str | None = None  # noqa: N815
+    referenceIdentifierField: str | None = None  # noqa: N815
 
 
 def loculus_organism_to_ena_organism(config: Config) -> dict[str, list[ENAOrganism]]:
     loculus_organism_to_ena_organism: dict[str, list[ENAOrganism]] = defaultdict(list)
     for ena_organism, details in config.enaOrganisms.items():
         if details.loculusOrganism:
-            if not details.suborganismIdentifierField:
+            if not details.referenceIdentifierField:
                 error_msg = (
-                    "Could not find suborganismIdentifierField in enaOrganism "
+                    "Could not find referenceIdentifierField in enaOrganism "
                     f"config for {ena_organism}"
                 )
                 logger.error(error_msg)
                 raise ValueError(error_msg) from None
             loculus_organism_to_ena_organism[details.loculusOrganism].append(
                 ENAOrganism(
                     enaOrganismName=ena_organism,
-                    suborganismIdentifierField=details.suborganismIdentifierField,
+                    referenceIdentifierField=details.referenceIdentifierField,
                 )
             )
             continue
@@ -94,11 +94,11 @@ def assign_ena_organism(
     entry: dict[str, Any],
     ena_organisms: list[ENAOrganism],
 ) -> str:
-    """Assign the correct ena organism based on suborganismIdentifierField if present."""
+    """Assign the correct ena organism based on referenceIdentifierField if present."""
     if len(ena_organisms) == 1:
         return ena_organisms[0].enaOrganismName
     for ena_organism in ena_organisms:
-        suborganism_field = ena_organism.suborganismIdentifierField
+        suborganism_field = ena_organism.referenceIdentifierField
         if (
             suborganism_field
             and entry["metadata"].get(suborganism_field) == ena_organism.enaOrganismName

ena-submission/src/ena_deposition/config.py

@@ -51,7 +51,7 @@ class EnaOrganismDetails(BaseModel):
     topology: Topology = Topology.LINEAR
     segments: list[str]
     loculusOrganism: str | None = None  # noqa: N815
-    suborganismIdentifierField: str | None = None  # noqa: N815
+    referenceIdentifierField: str | None = None  # noqa: N815
 
     def is_multi_segment(self) -> bool:
         return len(self.segments) > 1

kubernetes/loculus/templates/_common-metadata.tpl

@@ -304,8 +304,8 @@ organisms:
   {{- if .includeInDownloadsByDefault }}
   includeInDownloadsByDefault: {{ .includeInDownloadsByDefault }}
   {{- end }}
-  {{- if .onlyForSuborganism }}
-  onlyForSuborganism: {{ .onlyForSuborganism }}
+  {{- if .onlyForReference }}
+  onlyForReference: {{ .onlyForReference }}
   {{- end }}
   {{- if .customDisplay }}
   customDisplay:
@@ -331,7 +331,7 @@ organisms:
 
 {{/* Generate website metadata from passed metadata array */}}
 {{- define "loculus.generateWebsiteMetadata" }}
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" .referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" .referenceGenomes | fromYaml).segments }}
 {{- $isSegmented := gt (len $rawUniqueSegments) 1 }}
 {{- $metadataList := .metadata }}
 fields:
@@ -440,7 +440,7 @@ fields:
 
 {{/* Generate backend metadata from passed metadata array */}}
 {{- define "loculus.generateBackendMetadata" }}
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" .referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" .referenceGenomes | fromYaml).segments }}
 {{- $isSegmented := gt (len $rawUniqueSegments) 1 }}
 {{- $metadataList := .metadata }}
 fields:
@@ -464,7 +464,7 @@ fields:
 
 {{/* Generate backend metadata from passed metadata array */}}
 {{- define "loculus.generateBackendExternalMetadata" }}
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" .referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" .referenceGenomes | fromYaml).segments }}
 {{- $isSegmented := gt (len $rawUniqueSegments) 1 }}
 {{- $metadataList := .metadata }}
 fields:
@@ -527,11 +527,11 @@ enaOrganisms:
   {{- end }}
   {{- with $instance.schema }}
     {{ $configFile.configFile | toYaml | nindent 4 }}
-    {{- if $configFile.suborganismIdentifierField }}
-    suborganismIdentifierField: {{ quote $configFile.suborganismIdentifierField }}
+    {{- if $configFile.referenceIdentifierField }}
+    referenceIdentifierField: {{ quote $configFile.referenceIdentifierField }}
     {{- end }}
     organismName: {{ quote .organismName }}
-    {{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" $instance.referenceGenomes | fromYaml).segments }}
+    {{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" $instance.referenceGenomes | fromYaml).segments }}
     segments: {{ $rawUniqueSegments | toYaml | nindent 6 }}
     externalMetadata:
       {{- $args := dict

kubernetes/loculus/templates/_merged-reference-genomes.tpl

@@ -1,60 +1,70 @@
 {{- define "loculus.mergeReferenceGenomes" -}}
-{{- $referenceGenomes := . -}}
+{{- $segmentWithReferencesList := . -}}
 {{- $lapisNucleotideSequences := list -}}
 {{- $lapisGenes := list -}}
 
-{{- if len $referenceGenomes | eq 1 }}
-  {{- include "loculus.generateReferenceGenome" (first (values $referenceGenomes)) -}}
-{{- else }}
-  {{- range $suborganismName, $referenceGenomeRaw := $referenceGenomes -}}
-    {{- $referenceGenome := include "loculus.generateReferenceGenome" $referenceGenomeRaw | fromYaml -}}
-
-    {{- $nucleotideSequences := $referenceGenome.nucleotideSequences -}}
-    {{- if $nucleotideSequences -}}
-      {{- if eq (len $nucleotideSequences) 1 -}}
-        {{- $lapisNucleotideSequences = append $lapisNucleotideSequences (dict
-          "name" $suborganismName
-          "sequence" (first $nucleotideSequences).sequence)
-        -}}
-      {{- else -}}
-        {{- range $sequence := $nucleotideSequences -}}
-          {{- $lapisNucleotideSequences = append $lapisNucleotideSequences (dict
-            "name" (printf "%s-%s" $suborganismName $sequence.name)
-            "sequence" $sequence.sequence
-          ) -}}
-        {{- end -}}
-      {{- end -}}
+{{- if or (not $segmentWithReferencesList) (eq (len $segmentWithReferencesList) 0) -}}
+{{- $result := dict "nucleotideSequences" (list) "genes" (list) -}}
+{{- $result | toYaml -}}
+{{- else -}}
+
+{{- $singleSegment := eq (len $segmentWithReferencesList) 1 -}}
+
+{{- range $segment := $segmentWithReferencesList -}}
+  {{- $segmentName := $segment.name -}}
+  {{- $singleReference := eq (len $segment.references) 1 -}}
+  {{- range $reference := $segment.references -}}
+    {{- $referenceName := $reference.reference_name -}}
+    {{- if $singleReference -}}
+      {{/* Single reference mode - no suffix */}}
+      {{- $lapisNucleotideSequences = append $lapisNucleotideSequences (dict
+        "name" $segmentName
+        "sequence" $reference.sequence
+      ) -}}
+    {{- else -}}
+      {{- $name := printf "%s%s" (ternary "" (printf "%s-" $segmentName) $singleSegment) $referenceName -}}
+      {{- $lapisNucleotideSequences = append $lapisNucleotideSequences (dict
+        "name" $name
+        "sequence" $reference.sequence
+      ) -}}
     {{- end -}}
 
-    {{- if $referenceGenome.genes -}}
-      {{- range $gene := $referenceGenome.genes -}}
-        {{- $lapisGenes = append $lapisGenes (dict
-          "name" (printf "%s-%s" $suborganismName $gene.name)
-          "sequence" $gene.sequence)
-        -}}
+    {{/* Add genes if present */}}
+    {{- if $reference.genes -}}
+      {{- range $gene := $reference.genes -}}
+        {{- if $singleReference -}}
+          {{- $lapisGenes = append $lapisGenes (dict
+            "name" $gene.name
+            "sequence" $gene.sequence
+          ) -}}
+        {{- else -}}
+          {{- $geneName := printf "%s-%s" $gene.name $referenceName -}}
+          {{- $lapisGenes = append $lapisGenes (dict
+            "name" $geneName
+            "sequence" $gene.sequence
+          ) -}}
+        {{- end -}}
       {{- end -}}
     {{- end -}}
   {{- end -}}
-
-  {{- $result := dict "nucleotideSequences" $lapisNucleotideSequences "genes" $lapisGenes -}}
-  {{- $result | toYaml -}}
 {{- end -}}
 
 {{- end -}}
 
+{{- $result := dict "nucleotideSequences" $lapisNucleotideSequences "genes" $lapisGenes -}}
+{{- $result | toYaml -}}
+{{- end -}}
 
-{{- define "loculus.extractUniqueRawNucleotideSequenceNames" -}}
-{{- $referenceGenomes := . -}}
-{{- $segmentNames := list -}}
 
-{{- range $suborganismName, $referenceGenomeRaw := $referenceGenomes -}}
-  {{- $referenceGenome := include "loculus.generateReferenceGenome" $referenceGenomeRaw | fromYaml -}}
+{{- define "loculus.getNucleotideSegmentNames" -}}
+{{- $segmentWithReferencesList := . -}}
 
-  {{- range $sequence := $referenceGenome.nucleotideSequences -}}
-    {{- $segmentNames = append $segmentNames $sequence.name -}}
-  {{- end -}}
+{{/* Extract segment names directly from .name */}}
+{{- $segmentNames := list -}}
+{{- range $segment := $segmentWithReferencesList -}}
+  {{- $segmentNames = append $segmentNames $segment.name -}}
 {{- end -}}
 
 segments:
-{{- $segmentNames | uniq | toYaml | nindent 2 -}}
+{{- $segmentNames | sortAlpha | toYaml | nindent 2 -}}
 {{- end -}}

kubernetes/loculus/templates/_preprocessingFromValues.tpl

@@ -58,7 +58,7 @@
 {{- $metadata := .metadata }}
 {{- $referenceGenomes := .referenceGenomes}}
 
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" $referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" $referenceGenomes | fromYaml).segments }}
 {{- $isSegmented := gt (len $rawUniqueSegments) 1 }}
 
 {{- range $metadata }}

kubernetes/loculus/templates/_siloDatabaseConfig.tpl

@@ -13,7 +13,7 @@
 
 {{- define "loculus.siloDatabaseConfig" }}
 {{- $schema := .schema }}
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" .referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" .referenceGenomes | fromYaml).segments }}
 {{- $isSegmented := gt (len $rawUniqueSegments) 1 }}
 schema:
   instanceName: {{ $schema.organismName }}

kubernetes/loculus/templates/ingest-config.yaml

@@ -8,7 +8,7 @@
 {{- $values := $item.contents }}
 {{- if $values.ingest }}
 {{- $metadata := (include "loculus.patchMetadataSchema" $values.schema | fromYaml).metadata }}
-{{- $rawUniqueSegments := (include "loculus.extractUniqueRawNucleotideSequenceNames" $values.referenceGenomes | fromYaml).segments }}
+{{- $rawUniqueSegments := (include "loculus.getNucleotideSegmentNames" $values.referenceGenomes | fromYaml).segments }}
 ---
 apiVersion: v1
 kind: ConfigMap