epic: Allele centric mapping, storage, and service

Dockerfile

@@ -80,7 +80,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 -
 COPY poetry.lock pyproject.toml ./
 
 # installs runtime dependencies to $VIRTUAL_ENV
-RUN poetry install --no-root --extras server
+RUN poetry install --no-root --extras server --no-directory
 COPY alembic /code/alembic
 COPY alembic.ini /code/alembic.ini
 COPY src /code/src

Makefile

@@ -0,0 +1,23 @@
+VENV := .venv/bin
+
+.DEFAULT_GOAL := help
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  %-12s %s\n", $$1, $$2}'
+
+.PHONY: dev
+dev: ## Install deps including local editable variant-annotation
+	poetry install --extras server
+
+.PHONY: test
+test: ## Run the test suite
+	$(VENV)/pytest tests/
+
+.PHONY: lint
+lint: ## Check code with ruff
+	$(VENV)/ruff check src/ tests/
+
+.PHONY: format
+format: ## Format code with ruff
+	$(VENV)/ruff format src/ tests/

alembic/manual_migrations/migrate_target_gene_mapping_qc.py

@@ -76,7 +76,7 @@
 from mavedb.models import *  # noqa: F401,F403  pylint: disable=wildcard-import
 from mavedb.db.session import SessionLocal
 from mavedb.lib.variants import HGVS_G_REGEX, HGVS_P_REGEX, get_hgvs_from_post_mapped
-from mavedb.models.enums.annotation_layer import AnnotationLayer
+from mavedb.models.enums.sequence_level import SequenceLevel
 from mavedb.models.mapped_variant import MappedVariant
 from mavedb.models.score_set import ScoreSet
 from mavedb.models.target_gene import TargetGene
@@ -102,18 +102,18 @@
 _NUCLEOTIDE_ALPHABET = frozenset("ACGTUNRYSWKMBDHV-")
 
 
-def _layer_from_hgvs(hgvs: str) -> Optional[AnnotationLayer]:
-    """Classify an HGVS string into an :class:`AnnotationLayer`."""
+def _layer_from_hgvs(hgvs: str) -> Optional[SequenceLevel]:
+    """Classify an HGVS string into an :class:`SequenceLevel`."""
     if HGVS_G_REGEX.search(hgvs):
-        return AnnotationLayer.genomic
+        return SequenceLevel.genomic
     if HGVS_P_REGEX.search(hgvs):
-        return AnnotationLayer.protein
+        return SequenceLevel.protein
     if _HGVS_C_REGEX.search(hgvs):
-        return AnnotationLayer.cdna
+        return SequenceLevel.cdna
     return None
 
 
-def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[AnnotationLayer]:
+def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[SequenceLevel]:
     """
     Infer layer from VRS sequence alphabet (protein vs nucleotide).
     Returns protein if any allele/member sequence contains a non-nucleotide letter.
@@ -150,15 +150,15 @@ def extract_sequences(obj):
     for seq in extract_sequences(post_mapped):
         found_seq = True
         if any(letter.upper() not in _NUCLEOTIDE_ALPHABET for letter in seq if letter.isalpha()):
-            return AnnotationLayer.protein
+            return SequenceLevel.protein
 
     if found_seq:
-        return AnnotationLayer.genomic
+        return SequenceLevel.genomic
 
     return None
 
 
-def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]:
+def _alignment_level_for(target: TargetGene) -> Optional[SequenceLevel]:
     """Recover the alignment layer for legacy mapped variants of ``target``.
 
     Tries ``post_mapped_metadata`` first, then ``pre_mapped_metadata`` (same
@@ -177,16 +177,16 @@ def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]:
             return None
 
         if has_genomic:
-            return AnnotationLayer.genomic
+            return SequenceLevel.genomic
         if has_protein:
-            return AnnotationLayer.protein
+            return SequenceLevel.protein
         if has_cdna:
-            return AnnotationLayer.cdna
+            return SequenceLevel.cdna
 
     return None
 
 
-def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]:
+def _layer_from_sequence_type(target: TargetGene) -> Optional[SequenceLevel]:
     """Infer alignment layer from ``target_sequence.sequence_type`` and ``category``.
 
     Last-resort fallback for targets where all variant-level mappings failed
@@ -201,16 +201,16 @@ def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]:
         return None
     seq_type = target.target_sequence.sequence_type
     if seq_type == "protein":
-        return AnnotationLayer.protein
+        return SequenceLevel.protein
     if seq_type == "dna" and target.category is not None and target.category.value == "protein_coding":
-        return AnnotationLayer.genomic
+        return SequenceLevel.genomic
     return None
 
 
 def _populate_layer_by_target_id(
     db: Session,
     targets_by_score_set_id: dict[int, list[TargetGene]],
-    layer_by_target_id: dict[int, Optional[AnnotationLayer]],
+    layer_by_target_id: dict[int, Optional[SequenceLevel]],
     layer_via_hgvs_target_ids: set[int],
     layer_via_sequence_type_target_ids: set[int],
 ) -> None:
@@ -344,7 +344,7 @@ def do_migration(db: Session) -> None:
     # Cache (target_gene_id, alignment_level, tool_version) -> persisted TargetGeneMapping
     # so we create one row per distinct combination and reuse it across every
     # mapped variant that maps into it.
-    cache: dict[tuple[int, AnnotationLayer, str], TargetGeneMapping] = {}
+    cache: dict[tuple[int, SequenceLevel, str], TargetGeneMapping] = {}
 
     total = db.scalar(sa.select(sa.func.count(MappedVariant.id))) or 0
     print(f"  {total} mapped variants to consider.")
@@ -393,7 +393,7 @@ def do_migration(db: Session) -> None:
     # only need to determine it once per target -- and any single attributable mapped
     # variant for that target lets us attribute every sibling. The via_* sets track which
     # fallback was used so the diagnostic report shows attribution breadth.
-    layer_by_target_id: dict[int, Optional[AnnotationLayer]] = {}
+    layer_by_target_id: dict[int, Optional[SequenceLevel]] = {}
     layer_via_hgvs_target_ids: set[int] = set()
     layer_via_sequence_type_target_ids: set[int] = set()
     _populate_layer_by_target_id(
@@ -420,7 +420,7 @@ def do_migration(db: Session) -> None:
 
         # Group ids to update by (tgm_id, level) so we can issue one bulk UPDATE
         # per group instead of a per-row ORM flush.
-        updates_by_group: dict[tuple[int, AnnotationLayer], list[int]] = defaultdict(list)
+        updates_by_group: dict[tuple[int, SequenceLevel], list[int]] = defaultdict(list)
 
         for row in chunk:
             last_id = row.id

alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py

@@ -8,15 +8,18 @@
 
 from alembic import op
 
-from mavedb.models.published_variant import signature
-
 
 # revision identifiers, used by Alembic.
 revision = "4726e4dddde8"
 down_revision = "b85bc7b1bec7"
 branch_labels = None
 depends_on = None
 
+# Frozen literal, not imported from the model: this migration replaces the historical
+# mapped_variants-based MV indexes at its point in history. A later migration rebuilds the MV onto
+# the mapping_records substrate and re-indexes it, so the column names below stay the legacy ones.
+signature = "published_variants_materialized_view"
+
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###

alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py

@@ -0,0 +1,151 @@
+"""Add mapping_records, alleles, and mapping_record_alleles tables
+
+Revision ID: a1b2c3d4e5f6
+Revises: a7f3c2e9b104
+Create Date: 2026-05-29
+
+New parallel tables for the Better Reverse Translation epic (#746).
+The existing mapped_variants table is left untouched (frozen serving).
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a1b2c3d4e5f6"
+down_revision = "a7f3c2e9b104"
+branch_labels = None
+depends_on = None
+
+VALID_ASSAY_LEVELS = "('genomic', 'cdna', 'protein')"
+VALID_ALIGNMENT_LEVELS = "('protein', 'cdna', 'genomic')"
+
+
+def upgrade() -> None:
+    op.create_table(
+        "alleles",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("vrs_digest", sa.String(), nullable=False),
+        sa.Column("level", sa.String(length=16), nullable=False),
+        sa.Column("transcript", sa.String(), nullable=False),
+        sa.Column("hgvs_g", sa.String(), nullable=True),
+        sa.Column("hgvs_c", sa.String(), nullable=True),
+        sa.Column("hgvs_p", sa.String(), nullable=True),
+        sa.Column("clingen_allele_id", sa.String(), nullable=True),
+        sa.Column("post_mapped", postgresql.JSONB(), nullable=True),
+        sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")),
+        sa.Column(
+            "updated_at",
+            sa.Date(),
+            nullable=False,
+            server_default=sa.text("CURRENT_DATE"),
+            onupdate=sa.text("CURRENT_DATE"),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"),
+    )
+    op.create_index("ix_alleles_vrs_digest", "alleles", ["vrs_digest"])
+    op.create_index("ix_alleles_level", "alleles", ["level"])
+    op.create_index("ix_alleles_clingen_allele_id", "alleles", ["clingen_allele_id"])
+
+    op.create_table(
+        "mapping_records",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("variant_id", sa.Integer(), nullable=False),
+        sa.Column("vrs_digest", sa.String(), nullable=True),
+        sa.Column("pre_mapped", postgresql.JSONB(), nullable=True),
+        sa.Column("assay_level", sa.String(length=16), nullable=False),
+        sa.Column("hgvs_assay_level", sa.String(), nullable=True),
+        sa.Column("mapping_api_version", sa.String(), nullable=False),
+        sa.Column("mapped_date", sa.Date(), nullable=False),
+        sa.Column("vrs_version", sa.String(), nullable=True),
+        sa.Column("current", sa.Boolean(), nullable=False),
+        sa.Column("alignment_level", sa.String(length=16), nullable=True),
+        sa.Column("at_mismatched_locus", sa.Boolean(), nullable=True),
+        sa.Column("near_gap", sa.Boolean(), nullable=True),
+        sa.Column("target_gene_mapping_id", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")),
+        sa.Column(
+            "updated_at",
+            sa.Date(),
+            nullable=False,
+            server_default=sa.text("CURRENT_DATE"),
+            onupdate=sa.text("CURRENT_DATE"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["variant_id"],
+            ["variants.id"],
+            name="fk_mapping_records_variant_id",
+        ),
+        sa.ForeignKeyConstraint(
+            ["target_gene_mapping_id"],
+            ["target_gene_mappings.id"],
+            name="fk_mapping_records_target_gene_mapping_id",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.CheckConstraint(
+            f"assay_level IN {VALID_ASSAY_LEVELS}",
+            name="ck_mapping_records_assay_level_valid",
+        ),
+    )
+    op.create_index("ix_mapping_records_variant_id", "mapping_records", ["variant_id"])
+    op.create_index("ix_mapping_records_vrs_digest", "mapping_records", ["vrs_digest"])
+    op.create_index(
+        "ix_mapping_records_target_gene_mapping_id",
+        "mapping_records",
+        ["target_gene_mapping_id"],
+    )
+
+    op.create_table(
+        "mapping_record_alleles",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("mapping_record_id", sa.Integer(), nullable=False),
+        sa.Column("allele_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "is_authoritative",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["mapping_record_id"],
+            ["mapping_records.id"],
+            name="fk_mapping_record_alleles_mapping_record_id",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["allele_id"],
+            ["alleles.id"],
+            name="fk_mapping_record_alleles_allele_id",
+            ondelete="RESTRICT",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_mapping_record_alleles_mapping_record_id",
+        "mapping_record_alleles",
+        ["mapping_record_id"],
+    )
+    op.create_index(
+        "ix_mapping_record_alleles_allele_id",
+        "mapping_record_alleles",
+        ["allele_id"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_mapping_record_alleles_allele_id", table_name="mapping_record_alleles")
+    op.drop_index("ix_mapping_record_alleles_mapping_record_id", table_name="mapping_record_alleles")
+    op.drop_table("mapping_record_alleles")
+
+    op.drop_index("ix_mapping_records_target_gene_mapping_id", table_name="mapping_records")
+    op.drop_index("ix_mapping_records_vrs_digest", table_name="mapping_records")
+    op.drop_index("ix_mapping_records_variant_id", table_name="mapping_records")
+    op.drop_table("mapping_records")
+
+    op.drop_index("ix_alleles_clingen_allele_id", table_name="alleles")
+    op.drop_index("ix_alleles_level", table_name="alleles")
+    op.drop_index("ix_alleles_vrs_digest", table_name="alleles")
+    op.drop_table("alleles")