From 1297468b9b4c75673441814f9f80a6abbfc3a667 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 4 Jun 2026 11:05:29 -0700 Subject: [PATCH 01/93] feat(data-providers): add seqrepo-backed vrs data proxy - add shared seqrepo and seqrepo data proxy providers in services - reuse shared seqrepo provider in deps to remove duplicate config - align vrs sequence proxy behavior with dcd-mapping expectations --- src/mavedb/data_providers/services.py | 18 ++++++++++++++++++ src/mavedb/deps.py | 6 ++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/mavedb/data_providers/services.py b/src/mavedb/data_providers/services.py index a94c16d6e..45b91b756 100644 --- a/src/mavedb/data_providers/services.py +++ b/src/mavedb/data_providers/services.py @@ -2,7 +2,9 @@ from typing import TYPE_CHECKING, Optional import boto3 +from biocommons.seqrepo import SeqRepo from cdot.hgvs.dataproviders import ChainedSeqFetcher, FastaSeqFetcher, RESTDataProvider, SeqFetcher +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from mavedb.lib.mapping import VRSMap @@ -16,6 +18,7 @@ DCD_MAP_URL = os.environ.get("DCD_MAPPING_URL", "http://dcd-mapping:8000") CDOT_URL = os.environ.get("CDOT_URL", "http://cdot-rest:8000") +SEQREPO_DIR = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") CSV_UPLOAD_S3_BUCKET_NAME = os.getenv("UPLOAD_S3_BUCKET_NAME", "score-set-csv-uploads-dev") @@ -27,6 +30,21 @@ def cdot_rest() -> RESTDataProvider: return RESTDataProvider(url=CDOT_URL, seqfetcher=seqfetcher()) +def seqrepo() -> SeqRepo: + return SeqRepo(SEQREPO_DIR) + + +def seqrepo_data_proxy() -> SeqRepoDataProxy: + """VRS sequence/refget data proxy backed by local SeqRepo. + + Distinct from cdot_rest(): cdot is an hgvs *coordinate* data provider, while + AlleleTranslator needs a VRS *sequence* proxy that can derive_refget_accession. + Mirrors dcd_mapping's SeqRepo-backed translator. Uses the same HGVS_SEQREPO_DIR + location as deps.get_seqrepo and the refget router. + """ + return SeqRepoDataProxy(seqrepo()) + + def vrs_mapper(url: Optional[str] = None) -> VRSMap: return VRSMap(DCD_MAP_URL) if not url else VRSMap(url) diff --git a/src/mavedb/deps.py b/src/mavedb/deps.py index 8e4ce5106..712a6e5a3 100644 --- a/src/mavedb/deps.py +++ b/src/mavedb/deps.py @@ -1,4 +1,3 @@ -import os from typing import Any, AsyncGenerator, Generator from arq import ArqRedis, create_pool @@ -6,7 +5,7 @@ from cdot.hgvs.dataproviders import RESTDataProvider from sqlalchemy.orm import Session -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo from mavedb.db.session import SessionLocal from mavedb.worker.settings import RedisWorkerSettings @@ -33,5 +32,4 @@ def hgvs_data_provider() -> RESTDataProvider: def get_seqrepo() -> SeqRepo: - seqrepo_dir = os.environ.get("HGVS_SEQREPO_DIR", "/seqrepo") - return SeqRepo(seqrepo_dir) + return seqrepo() From 7ea450401d5c6134824fc4f980edd8bf1495431d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 4 Jun 2026 11:40:32 -0700 Subject: [PATCH 02/93] fix(vrs): centralize allele id recomputation helpers - add shared helpers to translate hgvs, normalize alleles, and compute ids - clear cached merkle digests before identification so mutated alleles do not keep stale ga4gh identifiers - document the invariant that all allele identification must route through the helper to prevent digest correctness regressions --- src/mavedb/lib/vrs_utils.py | 82 +++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/mavedb/lib/vrs_utils.py diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py new file mode 100644 index 000000000..2720afa68 --- /dev/null +++ b/src/mavedb/lib/vrs_utils.py @@ -0,0 +1,82 @@ +"""VRS allele identification helpers. + +Centralizes the digest-correctness invariant for GA4GH VRS alleles: the +``ga4gh_identify`` Merkle tree caches sub-object digests on the object after +first identification, so any subsequent mutation (refgetAccession swap, +normalization, state coercion) leaves a stale id unless the cached digests +are cleared first. All allele identification in dcd_mapping must route +through :func:`identify_allele` so the digest is always recomputed from +current content. +""" + +from typing import Any + +from ga4gh.core import ga4gh_identify +from ga4gh.vrs.extras.translator import AlleleTranslator +from ga4gh.vrs.models import Allele, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.normalize import normalize + + +def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: + """Convert HGVS variation description to VRS object. + + The AlleleTranslator is supplied by the caller and reused across calls. ga4gh's + AlleleTranslator opens a UTA connection lazily on first translate (via HgvsTools) + and holds it for its lifetime, so constructing one per call opens — and leaks — a + UTA connection per variant, exhausting the server's slot budget. Build one per + job/worker and pass it in. + + :param hgvs: MAVE-HGVS variation string + :param translator: caller-owned AlleleTranslator backed by a sequence/refget proxy + :return: Corresponding VRS allele as a Pydantic class + """ + # coerce tmp HGVS string into formally correct term + if hgvs.startswith("NC_") and ":c." in hgvs: + hgvs = hgvs.replace(":c.", ":g.") + + allele: Allele = translator.translate_from(hgvs, "hgvs", do_normalize=False) + + if ( + not isinstance(allele.location, SequenceLocation) + or not isinstance(allele.location.start, int) + or not isinstance(allele.location.end, int) + or not isinstance(allele.state, LiteralSequenceExpression) + ): + raise ValueError + + return allele + + +def identify_allele(allele: Allele) -> str: + """Clear cached digests and return a fresh GA4GH identifier for *allele*. + + ``ga4gh_identify`` is a Merkle-tree: it calls ``get_or_create_digest`` on + sub-objects, returning any cached value without recomputing. Clearing both + the location digest and the allele digest first ensures the id is always + derived from the current object content — not from a value set before a + refgetAccession mutation or normalization. + """ + if isinstance(allele.location, SequenceLocation): + allele.location.digest = None + + allele.digest = None + digest = ga4gh_identify(allele) + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 + + return digest + + +def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: + """Normalize *allele* and stamp it with a freshly computed GA4GH digest. + + Pairs the two finalize steps every VRS allele construction path needs. + Routing identification through :func:`identify_allele` (rather than + ``ga4gh_identify`` directly) is the invariant that protects against the + Merkle-tree's stale-digest behavior after mutation -- so any allele + construction site that bypasses this helper risks reintroducing the + stale-digest bug. + """ + allele = normalize(allele, data_proxy=data_proxy) + allele.id = identify_allele(allele) + return allele From 825cf9d23a7c180b8f8338731549202588052770 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:47:51 -0700 Subject: [PATCH 03/93] feat(db): add ValidTime mixin for valid-time row versioning Introduce a reusable declarative mixin implementing transaction-time SCD Type 2 versioning for rows that change over time, used by either versioned entities or link/association rows. - A row is live while valid_to is NULL; current/as_of express the half-open [valid_from, valid_to) point-in-time predicate so call sites never hand-roll it - supersede_with (single row) and supersede_live_where (bulk) stamp the retired valid_to and successor valid_from with one timestamp for a gap-free handoff regardless of transaction boundaries - retire/retire_live_where are withdrawal primitives; retire cascades to live child links named in __retire_cascade__ - bulk supersede refuses to run on cascade-bearing classes since it cannot fire the cascade - consumers must add a partial unique index over their natural key WHERE valid_to IS NULL as a backstop against duplicate live rows Cover the mixin with unit tests against purpose-built models, using explicit timestamps far from the transaction clock to prove the gap-free handoff and verify the partial unique index backstop. --- src/mavedb/db/mixins.py | 201 ++++++++++++++++++++++++++++ tests/db/test_mixins.py | 289 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 src/mavedb/db/mixins.py create mode 100644 tests/db/test_mixins.py diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py new file mode 100644 index 000000000..4bc5c1ca9 --- /dev/null +++ b/src/mavedb/db/mixins.py @@ -0,0 +1,201 @@ +"""Reusable declarative mixins.""" + +from datetime import datetime +from typing import ClassVar, Optional, Sequence, TypeVar + +from sqlalchemy import ColumnElement, DateTime, Update, func, inspect as sa_inspect, select, update +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, Session, mapped_column + +T = TypeVar("T", bound="ValidTime") + + +class ValidTime: + """Valid-time versioning for rows that change over time (transaction-time SCD Type 2). + + Applies to either a versioned entity (a row that is replaced by a newer version of the same + logical thing — e.g. a mapping record) or a link/association row (a relationship that comes + and goes). Immutable, content-addressed rows (deduplicated entities like an allele or a + source-versioned annotation record) do not use this — their "what applies now" is answered by + the links that reference them, not by versioning the row itself. + + A row is live while ``valid_to`` is NULL. Superseding a row sets its ``valid_to`` to the + successor's ``valid_from`` instead of deleting it, so the full history is retained and a + point-in-time query is a single half-open ``[valid_from, valid_to)`` predicate. ``current`` + and ``as_of`` express that predicate so call sites never hand-roll it. + + Convention: replacing a live row with a successor goes through :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk), which stamp the retired ``valid_to`` and the new + ``valid_from`` with one timestamp so the handoff is gap-free regardless of transaction + boundaries. ``retire`` (single) / ``retire_live_where`` (bulk) are *withdrawal* primitives — + retiring with no successor; do not pair a bare retire with a separate insert, or a slow job + between the two opens a window where the row is neither live-old nor live-new. The method pairs + mirror: ``retire``/``supersede_with`` act on one row, ``*_live_where`` act on a predicate. + + ``current`` is derived (``valid_to IS NULL``), not stored — one source of truth, no + dual-write to keep consistent. Each table using this mixin should add a partial unique index + over its natural key ``WHERE valid_to IS NULL`` to enforce a single live row per key (the link + key for an association, the logical-entity key for an entity). + + Transaction time only: ``valid_from``/``valid_to`` record when *we* held the row, not when an + external source considered it effective. Tables fed by externally-versioned sources (e.g. a + ClinVar release, a gnomAD version) should add their own source-version column for that axis. + + Consumer contract — a model mixing this in, and any code writing it, MUST honor: + + 1. **Add the partial unique index.** Declare a unique index over the natural key + ``WHERE valid_to IS NULL`` (one live row per key). It is the loud backstop: forgetting to + retire a predecessor before inserting its successor raises ``IntegrityError`` instead of + silently leaving two live rows. + 2. **Replace through supersede, never retire-then-insert.** Use :meth:`supersede_with` (single + row) or :meth:`supersede_live_where` (bulk) to swap a live row for a successor. A bare + :meth:`retire` followed by a separate insert is the gap footgun — if a slow job or a commit + falls between them, the retired ``valid_to`` and the new ``valid_from`` come from different + transaction clocks and a point-in-time query lands in a hole where the row is neither + old-live nor new-live. Reserve ``retire``/``retire_live_where`` for genuine withdrawal (no + successor). Gaps only arise on a *same-key* handoff; the cascade in (3) only ever retires + children under a superseded parent's old key, so it never needs a supersede. + 3. **Declare ``__retire_cascade__`` for parent rows with ``ValidTime`` children.** A live link to + a retired parent is stale; retiring a parent must retire its live child links. The ORM + relationship cascade does not do this (it fires only on hard DELETE). Bulk + :meth:`supersede_live_where` refuses to run on a class that declares ``__retire_cascade__`` + precisely because it cannot cascade — supersede such rows one at a time with + :meth:`supersede_with`. + 4. **Filter reads with ``current``/``as_of``.** Never read live rows by assuming a query returns + only current state; constrain with ``current`` (or ``as_of(ts)``) explicitly. Because of (3), + a live link implies a live parent, so allele/parent-side ``current`` queries do not surface + links dangling off retired parents. + """ + + valid_from: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + valid_to: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + + # Names of relationships to child ``ValidTime`` rows that should be retired when this row is. + # A link to a retired row is itself stale, so closing a parent's window must close its live + # links' windows too — the relationship cascade ("all, delete-orphan") only fires on a hard + # DELETE, not on this soft ``valid_to`` close. Each named relationship must target a + # ``ValidTime`` model; ``retire`` issues one bulk UPDATE per relationship. + # + # This covers only the transaction-time axis: "the parent was superseded, so its links are too." + # Links retired on a source-version axis (e.g. a new gnomAD release superseding an + # allele→gnomAD-variant link while the allele itself stays live) are a separate trigger and are + # not driven from here. Cascade is one level deep (bulk UPDATE, not per-row ``retire``); a deeper + # chain would need its own handling. + __retire_cascade__: ClassVar[tuple[str, ...]] = () + + @hybrid_property + def current(self) -> bool: + return self.valid_to is None + + @current.inplace.expression + @classmethod + def _current_expression(cls) -> ColumnElement[bool]: + return cls.valid_to.is_(None) + + @classmethod + def as_of(cls, ts: datetime) -> ColumnElement[bool]: + """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" + return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: + """Close this row's validity window, cascading to the live child links named in + ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. + + Pass ``at`` to stamp an explicit ``valid_to`` shared with a successor's ``valid_from``; this + is how :meth:`supersede_with` keeps the handoff gap-free. Without ``at`` the close uses + ``func.now()`` (Postgres ``transaction_timestamp``). This is a *withdrawal* primitive on its + own — to replace a row with a successor, use :meth:`supersede_with` so the handoff is closed. + + ``session`` is required only when ``__retire_cascade__`` is non-empty (the cascade issues a + bulk UPDATE per child relationship). A leaf row with no declared cascades retires without one. + The cascade runs even when this row was already closed, so a half-finished prior retire (row + closed, links not) still converges. + """ + stamp = at if at is not None else func.now() + if self.valid_to is None: + self.valid_to = stamp + + if not self.__retire_cascade__: + return + if session is None: + raise ValueError( + f"{type(self).__name__}.retire() needs a session to cascade-retire {self.__retire_cascade__}." + ) + + mapper = sa_inspect(type(self)) + assert mapper is not None # a mapped class always inspects to a Mapper + for name in self.__retire_cascade__: + rel = mapper.relationships[name] + child_cls = rel.mapper.class_ + conditions = [remote == getattr(self, local.key) for local, remote in rel.local_remote_pairs] + session.execute(child_cls.retire_live_where(*conditions, at=at)) + + @classmethod + def retire_live_where(cls, *conditions: ColumnElement[bool], at: Optional[datetime] = None) -> Update: + """An UPDATE that retires the live rows matching ``conditions`` (closes their valid_to). + Only currently-live rows are touched, so already-retired rows keep their original valid_to. + Pass ``at`` to stamp an explicit ``valid_to`` (used by :meth:`supersede`); defaults to + ``func.now()``. + + This is a *withdrawal* primitive — retiring live rows with no successor. To replace live rows + with new ones, use :meth:`supersede_live_where` so retire and insert share one timestamp. + + Usage: ``session.execute(Model.retire_live_where(Model.foo == bar))``. + """ + return ( + update(cls).where(cls.valid_to.is_(None), *conditions).values(valid_to=at if at is not None else func.now()) + ) + + def supersede_with(self: T, session: Session, replacement: T, at: Optional[datetime] = None) -> T: + """Retire this row (cascading to its child links) and insert ``replacement``, stamping the + retired row's ``valid_to`` and the replacement's ``valid_from`` with one timestamp so the + handoff has no gap — independent of how many transactions or how much wall-clock separate + them. The single-row counterpart to :meth:`supersede_live_where`, for superseding one known + row that may carry ``__retire_cascade__`` children. + + Flushes the retire before the insert: the partial unique index (one live row per natural key) + is checked per statement and the unit of work emits INSERTs before UPDATEs, so without the + flush the replacement and the not-yet-closed original would both be live and trip the index. + """ + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + self.retire(session, at=at) + session.flush() + replacement.valid_from = at + session.add(replacement) + return replacement + + @classmethod + def supersede_live_where( + cls, + session: Session, + new_rows: Sequence[T], + *conditions: ColumnElement[bool], + at: Optional[datetime] = None, + ) -> Sequence[T]: + """Retire the live rows matching ``conditions`` and insert ``new_rows``, stamping both sides + with one timestamp so every retired row's ``valid_to`` equals every new row's ``valid_from`` + — a gap-free handoff independent of transaction boundaries. The bulk counterpart to + :meth:`supersede_with` (and the supersede form of :meth:`retire_live_where`), for replacing a + *set* of live rows in one scope (e.g. a score set's derived links) with a freshly computed + set. + + The retire executes immediately (before the inserts flush), so a reused natural key does not + trip the partial unique index. Only for leaf ``ValidTime`` rows: a bulk retire cannot fire + ``__retire_cascade__``, so this refuses to run on a class that declares one — use + :meth:`supersede_with` per row there. + """ + if cls.__retire_cascade__: + raise ValueError( + f"{cls.__name__} declares __retire_cascade__; bulk supersede cannot cascade to children. " + "Use supersede_with per row so child links retire too." + ) + if at is None: + at = session.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + session.execute(cls.retire_live_where(*conditions, at=at)) + for row in new_rows: + row.valid_from = at + session.add_all(new_rows) + return new_rows diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py new file mode 100644 index 000000000..ad0333a5d --- /dev/null +++ b/tests/db/test_mixins.py @@ -0,0 +1,289 @@ +"""Unit tests for the ValidTime mixin. + +Tested against minimal, purpose-built models rather than any real MaveDB models. ``_VtParent`` is +a cascade-bearing entity (natural key ``key``); ``_VtChild`` is a leaf link (natural key +``(parent_id, tag)``). Both carry a partial unique index over their natural key +``WHERE valid_to IS NULL``, mirroring the contract real consumers must honor. + +Many checks pass an explicit ``at`` that differs from the DB clock: a single-transaction test cannot +otherwise distinguish a deliberate shared timestamp from the coincidental same-transaction handoff +the gap bug relied on, so the explicit ``at`` is what actually proves the timestamp is threaded. +""" + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import Column, ForeignKey, Index, Integer, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import declarative_base, relationship + +from mavedb.db.mixins import ValidTime + +VtBase = declarative_base() + +# Fixed, deterministic windows — far from the test's transaction clock so an accidental +# server_default (func.now()) stamp would be visibly wrong rather than coincidentally equal. +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +class _VtParent(ValidTime, VtBase): + __tablename__ = "vt_parents" + + id = Column(Integer, primary_key=True) + key = Column(Integer, nullable=False) + children = relationship("_VtChild", back_populates="parent") + + __retire_cascade__ = ("children",) + __table_args__ = (Index("uq_vt_parents_live", "key", unique=True, postgresql_where=text("valid_to IS NULL")),) + + +class _VtChild(ValidTime, VtBase): + __tablename__ = "vt_children" + + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey("vt_parents.id"), nullable=False) + tag = Column(Integer, nullable=False) + parent = relationship("_VtParent", back_populates="children") + + __table_args__ = ( + Index("uq_vt_children_live", "parent_id", "tag", unique=True, postgresql_where=text("valid_to IS NULL")), + ) + + +@pytest.fixture(autouse=True) +def vt_tables(session): + """Create the throwaway ValidTime tables on the test engine, drop them after. Autouse so every + test in the module gets the tables without naming the fixture in its signature.""" + engine = session.get_bind() + VtBase.metadata.create_all(bind=engine) + yield + # Release any locks the test's still-open transaction holds before dropping: a post-commit + # attribute access reopens a transaction holding ACCESS SHARE on these tables, and DROP TABLE + # would block on it forever (this fixture tears down while the session fixture is still open). + session.rollback() + VtBase.metadata.drop_all(bind=engine) + + +def _parent(session, key, *, valid_from=None): + p = _VtParent(key=key, valid_from=valid_from) if valid_from else _VtParent(key=key) + session.add(p) + session.commit() + return p + + +def _child(session, parent, tag, *, valid_from=None): + c = ( + _VtChild(parent_id=parent.id, tag=tag, valid_from=valid_from) + if valid_from + else _VtChild(parent_id=parent.id, tag=tag) + ) + session.add(c) + session.commit() + return c + + +class TestCurrentAndAsOf: + def test_fresh_row_is_current(self, session): + p = _parent(session, 1) + assert p.valid_to is None + assert p.current is True + + def test_retired_row_is_not_current(self, session): + p = _parent(session, 1) + p.retire(session, at=T1) + assert p.current is False + + def test_current_expression_filters_to_live_rows(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + session.commit() + + rows = session.scalars(select(_VtParent).where(_VtParent.current)).all() + assert [r.id for r in rows] == [live.id] + + def test_as_of_selects_rows_live_at_timestamp(self, session): + # Window [T0, T2): explicit valid_from and a retire at T2. + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def live_ids(ts): + return [r.id for r in session.scalars(select(_VtParent).where(_VtParent.as_of(ts))).all()] + + assert live_ids(T0 - timedelta(days=1)) == [] # before the window opens + assert live_ids(T0) == [p.id] # at valid_from (inclusive) + assert live_ids(T1) == [p.id] # inside the window + assert live_ids(T2) == [] # at valid_to (exclusive) + assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes + + +class TestRetire: + def test_retire_closes_valid_to(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + assert c.valid_to == T1 + + def test_retire_is_idempotent(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire(at=T1) + c.retire(at=T2) # already closed — keeps the original valid_to + assert c.valid_to == T1 + + def test_leaf_retire_needs_no_session(self, session): + c = _child(session, _parent(session, 1), 1) + c.retire() # no cascade declared, so no session required + session.commit() + assert c.valid_to is not None + + def test_retire_cascades_to_live_child_links(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + p.retire(session, at=T1) + session.commit() + + assert p.valid_to == T1 + assert c1.valid_to == T1 # cascade closes the children at the same instant + assert c2.valid_to == T1 + + def test_cascade_retire_without_session_raises(self, session): + p = _parent(session, 1) + with pytest.raises(ValueError, match="cascade-retire"): + p.retire() # cascade declared but no session to issue the child UPDATE + + def test_cascade_runs_even_when_row_already_closed(self, session): + # A half-finished prior retire (parent closed, child not) must still converge. + p = _parent(session, 1) + c = _child(session, p, 1) + p.valid_to = T0 # simulate parent closed without its child + session.commit() + + p.retire(session, at=T1) # idempotent on the parent, but still cascades + session.commit() + assert p.valid_to == T0 # unchanged (idempotent) + assert c.valid_to == T1 # child still retired + + +class TestRetireLiveWhere: + def test_retires_matching_live_rows(self, session): + p = _parent(session, 1) + c1 = _child(session, p, 1) + c2 = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert c1.valid_to == T1 + assert c2.valid_to == T1 + + def test_leaves_already_retired_rows_untouched(self, session): + p = _parent(session, 1) + already = _child(session, p, 1) + already.retire(at=T0) + session.commit() + live = _child(session, p, 2) + + session.execute(_VtChild.retire_live_where(_VtChild.parent_id == p.id, at=T1)) + session.commit() + + assert already.valid_to == T0 # retired row keeps its original valid_to + assert live.valid_to == T1 + + +class TestSupersedeWith: + def test_gap_free_handoff_with_explicit_at(self, session): + old = _parent(session, 1, valid_from=T0) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + # The retired valid_to equals the successor's valid_from exactly — no point-in-time hole. + assert old.valid_to == T1 + assert new.valid_from == T1 + + def test_default_at_handoff_is_gap_free(self, session): + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new) # at defaults to a single captured func.now() + session.commit() + + assert old.valid_to is not None + assert old.valid_to == new.valid_from + + def test_cascades_child_links_at_the_handoff_timestamp(self, session): + old = _parent(session, 1) + child = _child(session, old, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() + + assert old.valid_to == T1 + assert new.valid_from == T1 + assert child.valid_to == T1 # the old parent's child link retired under the same timestamp + + def test_respects_partial_unique_index_via_flush_ordering(self, session): + # old and new share natural key=1; supersede flushes the retire before inserting the + # successor, so the two are never simultaneously live and the unique index does not trip. + old = _parent(session, 1) + new = _VtParent(key=1) + + old.supersede_with(session, new, at=T1) + session.commit() # would raise IntegrityError if both were live at the INSERT + + live = session.scalars(select(_VtParent).where(_VtParent.key == 1, _VtParent.current)).all() + assert [r.id for r in live] == [new.id] + + +class TestSupersedeLiveWhere: + def test_stamps_explicit_valid_from_on_inserts(self, session): + # Without flushing, valid_from would be None under the server_default path; supersede sets it + # explicitly, which is what keeps the handoff gap-free across a later-transaction flush. + new = _VtChild(parent_id=1, tag=1) + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == 1, at=T1) + assert new.valid_from == T1 + + def test_gap_free_handoff_with_reused_natural_key(self, session): + p = _parent(session, 1) + old = _child(session, p, 1) + new = _VtChild(parent_id=p.id, tag=1) # same natural key (parent_id, tag) + + _VtChild.supersede_live_where(session, [new], _VtChild.parent_id == p.id, at=T1) + session.commit() # reused key: retire executes before the insert, so the index holds + + assert old.valid_to == T1 + assert new.valid_from == T1 + live = session.scalars(select(_VtChild).where(_VtChild.parent_id == p.id, _VtChild.current)).all() + assert [r.id for r in live] == [new.id] + + def test_empty_new_rows_retires_the_scope(self, session): + # A re-run that produces nothing should still withdraw the prior live set. + p = _parent(session, 1) + old = _child(session, p, 1) + + _VtChild.supersede_live_where(session, [], _VtChild.parent_id == p.id, at=T1) + session.commit() + + assert old.valid_to == T1 + assert session.scalars(select(_VtChild).where(_VtChild.current)).all() == [] + + def test_refuses_a_cascade_bearing_class(self, session): + # Bulk supersede cannot fire __retire_cascade__, so it must refuse rather than orphan links. + with pytest.raises(ValueError, match="__retire_cascade__"): + _VtParent.supersede_live_where(session, [], _VtParent.key == 1) + + +class TestPartialUniqueIndexBackstop: + def test_two_live_rows_for_one_key_raise(self, session): + # The DB index is the loud backstop: forgetting to retire a predecessor before inserting its + # successor fails at flush instead of silently leaving two live rows. + session.add(_VtParent(key=1)) + session.add(_VtParent(key=1)) + with pytest.raises(IntegrityError): + session.commit() From 160318002214792315ea3b2ba60f52497afcc47a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:48:38 -0700 Subject: [PATCH 04/93] feat(hgvs): add HGVS accession and cis-phased expression helpers Add a lib module with reusable helpers for parsing and rewriting HGVS strings ahead of VRS translation. - extract_accession returns the reference accession (substring before the first colon), tolerant of whitespace and missing separators - split_cis_phased_hgvs expands a bracketed multivariant expression into fully-qualified components carrying the original accession and coordinate prefix, since ga4gh's AlleleTranslator only yields a single Allele per call and each component must translate alone - join_cis_phased_hgvs is the inverse, recombining components into one bracketed block and returning None when they do not share a single accession and coordinate prefix Cover the helpers with unit tests including the split/join round trip and the mixed-accession and mixed-prefix rejection cases. --- src/mavedb/lib/hgvs.py | 80 ++++++++++++++++++++++++++++++++++++++++++ tests/lib/test_hgvs.py | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 src/mavedb/lib/hgvs.py create mode 100644 tests/lib/test_hgvs.py diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py new file mode 100644 index 000000000..f330bdfd4 --- /dev/null +++ b/src/mavedb/lib/hgvs.py @@ -0,0 +1,80 @@ +import re +from typing import Optional + +# Coordinate prefix of an HGVS variant description: a single type letter plus a dot +# (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. +_HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") + + +def extract_accession(hgvs_string: str) -> str: + """Extract the reference accession from an HGVS string, or return empty string if it cannot + be parsed. + + This function makes no assumptions about the structure of the accession (e.g. whether it starts with "NM_" + or "NC_") and is robust to extra whitespace. It simply returns the substring before the first colon, which + is the standard HGVS separator between the accession and the variant description. Callers must validate + the accession format separately if needed. + """ + token = (hgvs_string or "").strip() + if ":" not in token: + return "" + + return token.split(":", 1)[0].strip() + + +def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: + """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. + + The reverse-translate-variants tool emits the non-adjacent component substitutions of a + single codon change as one bracketed genomic expression (``NC_000001.11:g.[123A>G;125T>C]``) + whenever they are too far apart on the genome to collapse into a single delins. ga4gh's + AlleleTranslator only ever produces a single Allele, so each component must be translated + independently and recombined into a VRS CisPhasedBlock. + + Each returned component carries the original accession and coordinate prefix + (``NC_000001.11:g.123A>G``). A non-bracketed expression is returned unchanged as a + single-element list, so callers can treat both cases uniformly. + + Unlike a bare mavehgvs split, the accession is preserved: the components feed straight + into VRS translation, which requires a reference accession to resolve positions. + """ + if "[" not in hgvs_string: + return [hgvs_string] + + accession, _, remainder = hgvs_string.partition(":") + prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." + inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] + return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] + + +def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: + """Join cis-phased component HGVS strings into one bracketed expression. + + Inverse of :func:`split_cis_phased_hgvs`: ``["NC_…:g.123A>G", "NC_…:g.125T>C"]`` becomes + ``"NC_…:g.[123A>G;125T>C]"``. A single component is returned unchanged. + + Returns ``None`` when the components do not share a single accession and coordinate prefix — + they are then not expressible as one cis-phased block (e.g. members on different sequences). + """ + if not components: + return None + if len(components) == 1: + return components[0] + + accessions: set[str] = set() + prefixes: set[str] = set() + descriptions: list[str] = [] + for component in components: + accession, separator, remainder = component.partition(":") + match = _HGVS_COORD_PREFIX.match(remainder) + if not separator or not match: + return None + + accessions.add(accession) + prefixes.add(match.group(1)) + descriptions.append(match.group(2)) + + if len(accessions) != 1 or len(prefixes) != 1: + return None + + return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py new file mode 100644 index 000000000..355174fb1 --- /dev/null +++ b/tests/lib/test_hgvs.py @@ -0,0 +1,76 @@ +import pytest + +from mavedb.lib.hgvs import extract_accession, join_cis_phased_hgvs, split_cis_phased_hgvs + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NC_000001.11:g.1000A>G", "NC_000001.11"), + ("NM_000001.1:c.5A>G", "NM_000001.1"), + ("NC_000001.11:g.[1000A>G;1002T>C]", "NC_000001.11"), + ("no-colon-here", ""), + ("", ""), + ], +) +def test_extract_accession(hgvs, expected): + assert extract_accession(hgvs) == expected + + +def test_split_cis_phased_hgvs_passes_through_single_variant(): + assert split_cis_phased_hgvs("NC_000001.11:g.1000A>G") == ["NC_000001.11:g.1000A>G"] + + +def test_split_cis_phased_hgvs_splits_bracketed_genomic_expression(): + # Non-adjacent codon components are emitted as one bracketed genomic expression; each + # component must regain the accession and prefix to be VRS-translatable on its own. + assert split_cis_phased_hgvs("NC_000001.11:g.[1000A>G;1002T>C]") == [ + "NC_000001.11:g.1000A>G", + "NC_000001.11:g.1002T>C", + ] + + +def test_split_cis_phased_hgvs_handles_three_components_and_coding_prefix(): + assert split_cis_phased_hgvs("NM_000001.1:c.[1A>G;3T>C;5G>A]") == [ + "NM_000001.1:c.1A>G", + "NM_000001.1:c.3T>C", + "NM_000001.1:c.5G>A", + ] + + +def test_join_cis_phased_hgvs_passes_through_single_component(): + assert join_cis_phased_hgvs(["NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.1000A>G" + + +def test_join_cis_phased_hgvs_combines_genomic_components(): + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +@pytest.mark.parametrize( + "expression", + [ + "NC_000001.11:g.[1000A>G;1002T>C]", + "NM_000001.1:c.[1A>G;3T>C;5G>A]", + ], +) +def test_join_is_inverse_of_split(expression): + assert join_cis_phased_hgvs(split_cis_phased_hgvs(expression)) == expression + + +def test_join_cis_phased_hgvs_returns_none_for_empty(): + assert join_cis_phased_hgvs([]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_accessions(): + # Members on different sequences are not one cis-phased block. + assert join_cis_phased_hgvs(["NC_000001.11:g.1A>G", "NC_000002.12:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): + assert join_cis_phased_hgvs(["NM_000001.1:c.1A>G", "NM_000001.1:g.2T>C"]) is None + + +def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): + assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None From 0e6246413d65e5961057405fd96c8b2078866eac Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:49:11 -0700 Subject: [PATCH 05/93] feat(vrs): translate cis-phased multivariant HGVS into VRS blocks Add VRS translation support for cis-phased multivariant HGVS, building on the new hgvs split helpers. - translate_hgvs_to_variation translates each component HGVS to an Allele independently, returning a bare Allele for a single component and wrapping two or more in a CisPhasedBlock, mirroring dcd_mapping's vrs_map._construct_vrs_allele; the reverse-translation job emits bracketed genomic forms the AlleleTranslator cannot translate directly - identify_variation generalizes identify_allele to blocks, clearing every member and location digest plus the block's own before identifying so a stale member digest never propagates into the block id; the block digest is order-independent so a set dedups to one row Cover both with unit tests, including the order-independent block digest and the stale-digest clearing. --- src/mavedb/lib/vrs_utils.py | 55 +++++++++++++++++++++++- tests/lib/test_vrs_utils.py | 86 +++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 tests/lib/test_vrs_utils.py diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 2720afa68..7872f2ed6 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -13,9 +13,11 @@ from ga4gh.core import ga4gh_identify from ga4gh.vrs.extras.translator import AlleleTranslator -from ga4gh.vrs.models import Allele, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.models import Allele, CisPhasedBlock, LiteralSequenceExpression, SequenceLocation from ga4gh.vrs.normalize import normalize +from mavedb.lib.hgvs import split_cis_phased_hgvs + def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: """Convert HGVS variation description to VRS object. @@ -47,6 +49,32 @@ def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: return allele +def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Allele | CisPhasedBlock: + """Translate an HGVS expression — possibly a cis-phased multivariant — into a VRS object. + + Mirrors dcd_mapping's ``vrs_map._construct_vrs_allele``: each component HGVS is translated + to an Allele independently; a single component returns a bare Allele, while two or more are + wrapped in a CisPhasedBlock. The reverse-translation job emits bracketed genomic forms + (``g.[a;b]``) for non-adjacent codon components that ga4gh's AlleleTranslator cannot + translate directly, so splitting and recombining is the only way to represent them. + + The block's GA4GH digest is order-independent, so the same biological cis-phased set always + identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component + ordering. + + :param hgvs: a single- or cis-phased-multivariant HGVS string + :param translator: caller-owned AlleleTranslator reused across calls + :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set + """ + members = [translate_hgvs_to_vrs(component, translator) for component in split_cis_phased_hgvs(hgvs)] + if len(members) == 1: + return members[0] + + block = CisPhasedBlock(members=members) # type: ignore[call-arg] + block.id = identify_variation(block) + return block + + def identify_allele(allele: Allele) -> str: """Clear cached digests and return a fresh GA4GH identifier for *allele*. @@ -67,6 +95,31 @@ def identify_allele(allele: Allele) -> str: return digest +def identify_variation(variation: Allele | CisPhasedBlock) -> str: + """Clear cached digests and return a fresh GA4GH id for an Allele or CisPhasedBlock. + + Generalizes :func:`identify_allele` to cis-phased blocks. A block's Merkle digest is + derived from its members' digests, so a stale member digest would silently propagate into + the block id. Clear every member (and its location) plus the block itself before + identifying so the id always reflects current content. + """ + if isinstance(variation, Allele): + return identify_allele(variation) + + for member in variation.members: + if isinstance(member, Allele): + if isinstance(member.location, SequenceLocation): + member.location.digest = None + member.digest = None + + variation.digest = None + digest = ga4gh_identify(variation) + if digest is None: + raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 + + return digest + + def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """Normalize *allele* and stamp it with a freshly computed GA4GH digest. diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py new file mode 100644 index 000000000..6f2bd382f --- /dev/null +++ b/tests/lib/test_vrs_utils.py @@ -0,0 +1,86 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("ga4gh.vrs") + +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + SequenceLocation, + SequenceReference, +) + +from mavedb.lib import vrs_utils +from mavedb.lib.vrs_utils import identify_variation, translate_hgvs_to_variation + +_SQ = "SQ." + "a" * 32 + + +def _allele(start: int, alt: str) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + 1, + ), + state=LiteralSequenceExpression(sequence=alt), + ) + + +def _patch_component_translator(monkeypatch, alleles_by_hgvs: dict[str, Allele]) -> None: + """Stub the per-component single-allele translator that translate_hgvs_to_variation calls.""" + + def _fake(hgvs: str, translator=None) -> Allele: + return alleles_by_hgvs[hgvs] + + monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) + + +def test_single_variant_returns_a_bare_allele(monkeypatch): + allele = _allele(1000, "G") + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=None) + + assert isinstance(result, Allele) + assert result is allele + + +def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + + assert isinstance(result, CisPhasedBlock) + assert len(result.members) == 2 + assert result.id is not None and result.id.startswith("ga4gh:CPB.") + + +def test_cis_phased_block_digest_is_order_independent(monkeypatch): + members = { + "NC_000001.11:g.1000A>G": _allele(1000, "G"), + "NC_000001.11:g.1002T>C": _allele(1002, "C"), + } + _patch_component_translator(monkeypatch, members) + + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=None) + + # The same biological cis-phased set dedups to one row regardless of component ordering. + assert forward.id == reverse.id + + +def test_identify_variation_clears_stale_block_digest(): + block = CisPhasedBlock(members=[_allele(1000, "G"), _allele(1002, "C")]) + block.digest = "STALE" + + digest = identify_variation(block) + + assert digest != "STALE" + assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache From ce510c33d527aa9e4cf1fca3ce472116676cd913 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 06/93] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- src/mavedb/lib/score_sets.py | 8 ++++++-- src/mavedb/lib/variants.py | 17 +++++++++++------ .../worker/jobs/external_services/clingen.py | 5 +++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..2acfa26f7 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -906,7 +906,9 @@ def variant_to_csv_row( value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep if value == na_rep: fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) + if mapping and mapping.post_mapped + else None ) if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): value = fallback_hgvs @@ -917,7 +919,9 @@ def variant_to_csv_row( value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep if value == na_rep: fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) + if mapping and mapping.post_mapped + else None ) if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): value = fallback_hgvs diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index bef9725c0..a941006cf 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -1,6 +1,7 @@ import re from typing import Any, Optional +from mavedb.lib.hgvs import join_cis_phased_hgvs from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant @@ -25,7 +26,15 @@ def hgvs_from_vrs_allele(allele: dict) -> str: raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") -def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: +def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: + """Extract a single HGVS string from a post-mapped VRS object. + + Multi-variant blocks (Haplotype/CisPhasedBlock) are cis-phased, so their members combine + into one bracketed expression (``NC_…:g.[a;b]``) when ``combine_cis`` is set. It defaults + off because some consumers cannot yet handle a bracketed expression — notably ClinGen + submission, which has no single CAID for a multi-variant cis block (see + https://github.com/VariantEffect/mavedb-api/issues/764). + """ if not post_mapped_vrs: return None @@ -40,13 +49,9 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: if len(variations_hgvs) == 0: return None - # raise ValueError(f"No variations found in variant {variant_urn}.") - # TODO (https://github.com/VariantEffect/mavedb-api/issues/468) In a future version, we will be able to generate - # a combined HGVS string for haplotypes and cis phased blocks directly from mapper output. if len(variations_hgvs) > 1: - return None - # raise ValueError(f"Multiple variations found in variant {variant_urn}.") + return join_cis_phased_hgvs(variations_hgvs) if combine_cis else None return variations_hgvs[0] diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 639160ed5..5a65cc728 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -137,6 +137,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: variant_post_mapped_hgvs: dict[str, list[int]] = {} no_hgvs_count = 0 for mapped_variant_id, post_mapped in variant_post_mapped_objects: + # Intentionally not combine_cis=True: multi-variant cis-phased blocks have no single + # CAID, so they are skipped here pending ClinGen guidance on how to register them + # (https://github.com/VariantEffect/mavedb-api/issues/764). hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) if not hgvs_for_post_mapped: @@ -357,6 +360,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: variant_content = [] variant_for_urn = {} for variant, mapped_variant in variant_objects: + # See the note above: cis-phased blocks are skipped here pending ClinGen guidance + # (https://github.com/VariantEffect/mavedb-api/issues/764). variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) if not variation: From 058ff3f293d5203783e94aa176cc7f07ffcf8e5d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:54:54 -0700 Subject: [PATCH 07/93] feat(db): add VRS allele closure tables for reverse translation Introduce parallel mapping tables for the Better Reverse Translation epic (#746). The existing mapped_variants table is left untouched (frozen serving) while the new schema is built out separately. - add alleles, mapping_records, and mapping_record_alleles tables with their ORM models; alleles are content-addressed by vrs_digest and shared across mapping records via the link table - mapping_record_alleles.is_authoritative distinguishes the assay's actual measurement from translator-derived links, since the same VRS allele can be authoritative for one record and derived for another - put mapping_records and mapping_record_alleles on valid-time versioning (ValidTime mixin): a re-map retires the prior live row by closing valid_to instead of deleting, so history is retained and point-in-time queries are a single predicate; partial unique indexes promote "one live row per key" to the database - derive Allele.transcript and MappingRecord.transcript as hybrid properties from the HGVS columns rather than storing them, so they cannot drift; drop the stored alleles.transcript column - add the cross_level_translation annotation type, written once per variant to record whether filling unmapped levels succeeded, was skipped, or failed - annotate Variant and TargetGeneMapping with mapping_records relationships and typed primary keys --- ...2c3d4e5f6_add_vrs_allele_closure_tables.py | 151 ++++++++++++++++++ ...f0a2c4d7_drop_alleles_transcript_column.py | 32 ++++ ...7f9a1b2_temporal_mapping_record_alleles.py | 48 ++++++ .../d4e6f8a0b2c3_temporal_mapping_records.py | 76 +++++++++ ...cross_level_translation_annotation_type.py | 49 ++++++ src/mavedb/models/__init__.py | 3 + src/mavedb/models/allele.py | 59 +++++++ src/mavedb/models/enums/annotation_type.py | 1 + src/mavedb/models/mapping_record.py | 96 +++++++++++ src/mavedb/models/mapping_record_allele.py | 60 +++++++ src/mavedb/models/target_gene_mapping.py | 10 +- src/mavedb/models/variant.py | 7 +- .../models/variant_annotation_status.py | 2 +- 13 files changed, 590 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py create mode 100644 alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py create mode 100644 alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py create mode 100644 alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py create mode 100644 alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py create mode 100644 src/mavedb/models/allele.py create mode 100644 src/mavedb/models/mapping_record.py create mode 100644 src/mavedb/models/mapping_record_allele.py diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py new file mode 100644 index 000000000..49c502d05 --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py @@ -0,0 +1,151 @@ +"""Add mapping_records, alleles, and mapping_record_alleles tables + +Revision ID: a1b2c3d4e5f6 +Revises: 398067c53257 +Create Date: 2026-05-29 + +New parallel tables for the Better Reverse Translation epic (#746). +The existing mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "398067c53257" +branch_labels = None +depends_on = None + +VALID_ASSAY_LEVELS = "('genomic', 'cdna', 'protein')" +VALID_ALIGNMENT_LEVELS = "('protein', 'cdna', 'genomic')" + + +def upgrade() -> None: + op.create_table( + "alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=False), + sa.Column("level", sa.String(length=16), nullable=False), + sa.Column("transcript", sa.String(), nullable=False), + sa.Column("hgvs_g", sa.String(), nullable=True), + sa.Column("hgvs_c", sa.String(), nullable=True), + sa.Column("hgvs_p", sa.String(), nullable=True), + sa.Column("clingen_allele_id", sa.String(), nullable=True), + sa.Column("post_mapped", postgresql.JSONB(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + ) + op.create_index("ix_alleles_vrs_digest", "alleles", ["vrs_digest"]) + op.create_index("ix_alleles_level", "alleles", ["level"]) + op.create_index("ix_alleles_clingen_allele_id", "alleles", ["clingen_allele_id"]) + + op.create_table( + "mapping_records", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=False), + sa.Column("vrs_digest", sa.String(), nullable=True), + sa.Column("pre_mapped", postgresql.JSONB(), nullable=True), + sa.Column("assay_level", sa.String(length=16), nullable=False), + sa.Column("hgvs_assay_level", sa.String(), nullable=True), + sa.Column("mapping_api_version", sa.String(), nullable=False), + sa.Column("mapped_date", sa.Date(), nullable=False), + sa.Column("vrs_version", sa.String(), nullable=True), + sa.Column("current", sa.Boolean(), nullable=False), + sa.Column("alignment_level", sa.String(length=16), nullable=True), + sa.Column("at_mismatched_locus", sa.Boolean(), nullable=True), + sa.Column("near_gap", sa.Boolean(), nullable=True), + sa.Column("target_gene_mapping_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.Date(), nullable=False, server_default=sa.text("CURRENT_DATE")), + sa.Column( + "updated_at", + sa.Date(), + nullable=False, + server_default=sa.text("CURRENT_DATE"), + onupdate=sa.text("CURRENT_DATE"), + ), + sa.ForeignKeyConstraint( + ["variant_id"], + ["variants.id"], + name="fk_mapping_records_variant_id", + ), + sa.ForeignKeyConstraint( + ["target_gene_mapping_id"], + ["target_gene_mappings.id"], + name="fk_mapping_records_target_gene_mapping_id", + ), + sa.PrimaryKeyConstraint("id"), + sa.CheckConstraint( + f"assay_level IN {VALID_ASSAY_LEVELS}", + name="ck_mapping_records_assay_level_valid", + ), + ) + op.create_index("ix_mapping_records_variant_id", "mapping_records", ["variant_id"]) + op.create_index("ix_mapping_records_vrs_digest", "mapping_records", ["vrs_digest"]) + op.create_index( + "ix_mapping_records_target_gene_mapping_id", + "mapping_records", + ["target_gene_mapping_id"], + ) + + op.create_table( + "mapping_record_alleles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("mapping_record_id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column( + "is_authoritative", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.ForeignKeyConstraint( + ["mapping_record_id"], + ["mapping_records.id"], + name="fk_mapping_record_alleles_mapping_record_id", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_mapping_record_alleles_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_alleles", + ["mapping_record_id"], + ) + op.create_index( + "ix_mapping_record_alleles_allele_id", + "mapping_record_alleles", + ["allele_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_mapping_record_alleles_allele_id", table_name="mapping_record_alleles") + op.drop_index("ix_mapping_record_alleles_mapping_record_id", table_name="mapping_record_alleles") + op.drop_table("mapping_record_alleles") + + op.drop_index("ix_mapping_records_target_gene_mapping_id", table_name="mapping_records") + op.drop_index("ix_mapping_records_vrs_digest", table_name="mapping_records") + op.drop_index("ix_mapping_records_variant_id", table_name="mapping_records") + op.drop_table("mapping_records") + + op.drop_index("ix_alleles_clingen_allele_id", table_name="alleles") + op.drop_index("ix_alleles_level", table_name="alleles") + op.drop_index("ix_alleles_vrs_digest", table_name="alleles") + op.drop_table("alleles") diff --git a/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py new file mode 100644 index 000000000..2705d9a3c --- /dev/null +++ b/alembic/versions/b8e1f0a2c4d7_drop_alleles_transcript_column.py @@ -0,0 +1,32 @@ +"""drop alleles.transcript column + +Revision ID: b8e1f0a2c4d7 +Revises: f4d2a9c1b7e3 +Create Date: 2026-06-05 + +The `transcript` column duplicated data already present in the HGVS columns — it was +always extract_accession(hgvs_g/hgvs_c/hgvs_p). It is now a derived hybrid_property on +the Allele model (split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)), so the stored +column is removed to keep a single source of truth and avoid drift. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b8e1f0a2c4d7" +down_revision = "f4d2a9c1b7e3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_column("alleles", "transcript") + + +def downgrade() -> None: + # Re-add the column and backfill it from the HGVS columns (the same derivation the + # hybrid_property uses) so the restored NOT NULL column is consistent. + op.add_column("alleles", sa.Column("transcript", sa.String(), nullable=True)) + op.execute("UPDATE alleles SET transcript = split_part(coalesce(hgvs_g, hgvs_c, hgvs_p), ':', 1)") + op.alter_column("alleles", "transcript", nullable=False) diff --git a/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py new file mode 100644 index 000000000..26d3d6dbf --- /dev/null +++ b/alembic/versions/c3d5e7f9a1b2_temporal_mapping_record_alleles.py @@ -0,0 +1,48 @@ +"""add valid-time versioning to mapping_record_alleles + +Revision ID: c3d5e7f9a1b2 +Revises: b8e1f0a2c4d7 +Create Date: 2026-06-05 + +Make the link table valid-time versioned (TemporalLink): a link is live while valid_to is +NULL, and superseding it closes valid_to instead of deleting, so reverse translation can be +re-run independently while prior derivations remain queryable point-in-time. The partial +unique index enforces a single live link per (mapping_record, allele). + +Assumes no pre-existing duplicate live links — true for these parallel tables, which are +new-only writes and not yet serving. If this ever runs against data with duplicates, the +unique index creation will fail and the duplicates must be retired first. +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c3d5e7f9a1b2" +down_revision = "b8e1f0a2c4d7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.add_column( + "mapping_record_alleles", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "uq_mapping_record_alleles_live", + "mapping_record_alleles", + ["mapping_record_id", "allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live", table_name="mapping_record_alleles") + op.drop_column("mapping_record_alleles", "valid_to") + op.drop_column("mapping_record_alleles", "valid_from") diff --git a/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py new file mode 100644 index 000000000..603c171a5 --- /dev/null +++ b/alembic/versions/d4e6f8a0b2c3_temporal_mapping_records.py @@ -0,0 +1,76 @@ +"""move mapping_records onto valid-time versioning + +Revision ID: d4e6f8a0b2c3 +Revises: c3d5e7f9a1b2 +Create Date: 2026-06-05 + +Replace the stored `current` flag and the `created_at`/`updated_at` audit dates with valid-time +columns (ValidTime mixin): a mapping record is live while valid_to is NULL, and a re-map retires +the prior version (closing valid_to) instead of flipping a boolean. `current` becomes derived +(valid_to IS NULL). `mapped_date` (the date the mapping was performed) is domain data and stays. + +The partial unique index promotes to the database the "one live mapping record per variant" +invariant the mapping job previously enforced only in app code. + +Backfills from the columns being dropped, so existing rows keep their validity. Assumes no +duplicate live records per variant (true for these pre-cutover parallel tables; otherwise the +unique index creation fails and the duplicates must be retired first). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d4e6f8a0b2c3" +down_revision = "c3d5e7f9a1b2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_records", + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=True, server_default=sa.func.now()), + ) + op.add_column( + "mapping_records", + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + ) + + # Backfill validity from the columns being replaced: a record's life began at created_at, and + # a non-current record was retired at updated_at (the only in-place update it ever took). + op.execute("UPDATE mapping_records SET valid_from = created_at::timestamptz") + op.execute("UPDATE mapping_records SET valid_to = updated_at::timestamptz WHERE current = false") + + op.alter_column("mapping_records", "valid_from", nullable=False) + + op.drop_column("mapping_records", "current") + op.drop_column("mapping_records", "created_at") + op.drop_column("mapping_records", "updated_at") + + op.create_index( + "uq_mapping_records_current", + "mapping_records", + ["variant_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_records_current", table_name="mapping_records") + + op.add_column("mapping_records", sa.Column("current", sa.Boolean(), nullable=True)) + op.add_column("mapping_records", sa.Column("created_at", sa.Date(), nullable=True)) + op.add_column("mapping_records", sa.Column("updated_at", sa.Date(), nullable=True)) + + op.execute("UPDATE mapping_records SET current = (valid_to IS NULL)") + op.execute("UPDATE mapping_records SET created_at = valid_from::date") + op.execute("UPDATE mapping_records SET updated_at = coalesce(valid_to, valid_from)::date") + + op.alter_column("mapping_records", "current", nullable=False) + op.alter_column("mapping_records", "created_at", nullable=False) + op.alter_column("mapping_records", "updated_at", nullable=False) + + op.drop_column("mapping_records", "valid_to") + op.drop_column("mapping_records", "valid_from") diff --git a/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py new file mode 100644 index 000000000..790659d08 --- /dev/null +++ b/alembic/versions/f4d2a9c1b7e3_add_cross_level_translation_annotation_type.py @@ -0,0 +1,49 @@ +"""add cross_level_translation annotation type + +Revision ID: f4d2a9c1b7e3 +Revises: a1b2c3d4e5f6 +Create Date: 2026-06-02 + +Extends ck_variant_annotation_type_valid to allow the 'cross_level_translation' +annotation type. The VRS mapping worker writes one such row per variant to record +whether cross-level translation (filling the levels the assay did not map) +succeeded, was skipped (multivariant / no transcript), or failed. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f4d2a9c1b7e3" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + +_TYPES_OLD = ( + "'vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) +_TYPES_NEW = "'vrs_mapping', 'cross_level_translation', " + ( + "'clingen_allele_id', 'mapped_hgvs', 'variant_translation', " + "'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', " + "'ldh_submission'" +) + + +def upgrade() -> None: + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_NEW})", + ) + + +def downgrade() -> None: + op.execute("DELETE FROM variant_annotation_status WHERE annotation_type = 'cross_level_translation'") + op.drop_constraint("ck_variant_annotation_type_valid", "variant_annotation_status", type_="check") + op.create_check_constraint( + "ck_variant_annotation_type_valid", + "variant_annotation_status", + f"annotation_type IN ({_TYPES_OLD})", + ) diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 2f0d65b48..35c1ba0a5 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -1,6 +1,7 @@ __all__ = [ "access_key", "acmg_classification", + "allele", "collection", "clinical_control", "controlled_keyword", @@ -16,6 +17,8 @@ "legacy_keyword", "license", "mapped_variant", + "mapping_record", + "mapping_record_allele", "pipeline", "publication_identifier", "published_variant", diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py new file mode 100644 index 000000000..54ce07e50 --- /dev/null +++ b/src/mavedb/models/allele.py @@ -0,0 +1,59 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Column, Date, Index, Integer, String, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.lib.hgvs import extract_accession + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + + +class Allele(Base): + __tablename__ = "alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + + vrs_digest = Column(String, nullable=False) + level = Column(String(length=16), nullable=False) + + hgvs_g = Column(String, nullable=True) + hgvs_c = Column(String, nullable=True) + hgvs_p = Column(String, nullable=True) + + clingen_allele_id = Column(String, nullable=True) + post_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + created_at = Column(Date, nullable=False, default=date.today) + updated_at = Column(Date, nullable=False, default=date.today, onupdate=date.today) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the populated HGVS column (derived, not stored). + + Exactly one of hgvs_g/hgvs_c/hgvs_p is populated per allele, and the transcript is that + string's accession. Derived rather than stored so it cannot drift from the HGVS column + it duplicates. + """ + return extract_accession(self.hgvs_g or self.hgvs_c or self.hgvs_p or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(func.coalesce(cls.hgvs_g, cls.hgvs_c, cls.hgvs_p), ":", 1) + + mapping_record_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="allele", + ) + + __table_args__ = ( + UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), + Index("ix_alleles_vrs_digest", "vrs_digest"), + Index("ix_alleles_level", "level"), + Index("ix_alleles_clingen_allele_id", "clingen_allele_id"), + ) diff --git a/src/mavedb/models/enums/annotation_type.py b/src/mavedb/models/enums/annotation_type.py index b1595347b..a3bbd1693 100644 --- a/src/mavedb/models/enums/annotation_type.py +++ b/src/mavedb/models/enums/annotation_type.py @@ -3,6 +3,7 @@ class AnnotationType(str, Enum): VRS_MAPPING = "vrs_mapping" + CROSS_LEVEL_TRANSLATION = "cross_level_translation" CLINGEN_ALLELE_ID = "clingen_allele_id" MAPPED_HGVS = "mapped_hgvs" VARIANT_TRANSLATION = "variant_translation" diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py new file mode 100644 index 000000000..132a69b8b --- /dev/null +++ b/src/mavedb/models/mapping_record.py @@ -0,0 +1,96 @@ +from datetime import date +from typing import TYPE_CHECKING, Any, Optional + +from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime +from mavedb.lib.hgvs import extract_accession +from mavedb.models.enums.annotation_layer import AnnotationLayer + +if TYPE_CHECKING: + from .mapping_record_allele import MappingRecordAllele + from .target_gene_mapping import TargetGeneMapping + from .variant import Variant + + +class MappingRecord(ValidTime, Base): + __tablename__ = "mapping_records" + + id: Mapped[int] = Column(Integer, primary_key=True) + + variant_id = Column(Integer, ForeignKey("variants.id"), nullable=False) + variant: Mapped["Variant"] = relationship("Variant", back_populates="mapping_records") + + # Digest of the pre-mapped (assayed-level) VRS representation, indexed for + # cross-score-set dedup lookups. + vrs_digest = Column(String, nullable=True) + pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) + + # Level at which the variant was *assayed* — distinct from alignment_level (QC). + assay_level = Column(String(length=16), nullable=False) + hgvs_assay_level = Column(String, nullable=True) + + @hybrid_property + def transcript(self) -> str: + """Reference accession of the assay-level HGVS (derived, not stored). + + Derived rather than stored so it cannot drift from hgvs_assay_level, which already + carries the accession. + """ + return extract_accession(self.hgvs_assay_level or "") + + @transcript.inplace.expression + @classmethod + def _transcript_expression(cls): + return func.split_part(cls.hgvs_assay_level, ":", 1) + + mapping_api_version = Column(String, nullable=False) + # Domain data: the date the mapping was performed (from the dcd-mapping run), surfaced to + # users — distinct from valid_from, which is when this version became the live record. + mapped_date = Column(Date, nullable=False, default=date.today) + vrs_version = Column(String, nullable=True) + + # valid_from / valid_to and the derived `current` come from ValidTime: a re-map retires the + # prior live record (closes its valid_to) and inserts a new live one, so mapping history is + # retained and point-in-time queries are a single predicate. + + # Per-mapping QC fields from dcd-mapping. + alignment_level = Column( + Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + nullable=True, + ) + at_mismatched_locus = Column(Boolean, nullable=True) + near_gap = Column(Boolean, nullable=True) + + target_gene_mapping_id = Column(Integer, ForeignKey("target_gene_mappings.id"), index=True, nullable=True) + target_gene_mapping: Mapped[Optional["TargetGeneMapping"]] = relationship( + "TargetGeneMapping", back_populates="mapping_records" + ) + + allele_links: Mapped[list["MappingRecordAllele"]] = relationship( + "MappingRecordAllele", + back_populates="mapping_record", + cascade="all, delete-orphan", + ) + + # Retiring a record retires its live allele links too (both the authoritative link and any + # derived links a reverse-translation run attached). See ValidTime.__retire_cascade__. + __retire_cascade__ = ("allele_links",) + + __table_args__ = ( + Index("ix_mapping_records_vrs_digest", "vrs_digest"), + Index("ix_mapping_records_variant_id", "variant_id"), + # At most one live mapping record per variant — promotes to the database the invariant the + # mapping job enforces in app code (it retires the prior live record before inserting a new + # one). Superseded versions remain with a closed valid_to for point-in-time queries. + Index( + "uq_mapping_records_current", + "variant_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py new file mode 100644 index 000000000..cda8bf0f3 --- /dev/null +++ b/src/mavedb/models/mapping_record_allele.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .mapping_record import MappingRecord + + +class MappingRecordAllele(ValidTime, Base): + __tablename__ = "mapping_record_alleles" + + id: Mapped[int] = Column(Integer, primary_key=True) + mapping_record_id: Mapped[int] = Column( + Integer, + ForeignKey("mapping_records.id", ondelete="CASCADE"), + nullable=False, + ) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + # Provenance: ``True`` when this allele was the assay's actual measurement + # for this mapping record (sourced from vrs_map, with real alignment QC), + # ``False`` when it was derived by the cross-level or same-level + # synonymous translator. The flag lives on the link rather than on + # ``Allele`` because the same VRS allele can be authoritative for one + # mapping record and translator-derived for another. + is_authoritative = Column(Boolean, nullable=False, default=False) + + mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") + allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") + + __table_args__ = ( + Index( + "ix_mapping_record_alleles_mapping_record_id", + "mapping_record_id", + ), + Index( + "ix_mapping_record_alleles_allele_id", + "allele_id", + ), + # At most one live link per (mapping_record, allele). The job retires (closes valid_to) + # rather than deletes, so superseded links remain for point-in-time queries; only the + # live row participates in this constraint. A derived link is never written when an + # authoritative one already exists for the pair, so is_authoritative is intentionally + # absent from the key — a record links a given allele once. + Index( + "uq_mapping_record_alleles_live", + "mapping_record_id", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 417f249d4..52761e34b 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -23,15 +23,16 @@ if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant + from mavedb.models.mapping_record import MappingRecord from mavedb.models.target_gene import TargetGene class TargetGeneMapping(Base): __tablename__ = "target_gene_mappings" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) - target_gene_id = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) + target_gene_id: Mapped[int] = Column(Integer, ForeignKey("target_genes.id"), nullable=False, index=True) target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( @@ -80,4 +81,9 @@ class TargetGeneMapping(Base): back_populates="target_gene_mapping", ) + mapping_records: Mapped[list["MappingRecord"]] = relationship( + "MappingRecord", + back_populates="target_gene_mapping", + ) + __table_args__ = (Index("ix_target_gene_mappings_target_alignment", "target_gene_id", "alignment_level"),) diff --git a/src/mavedb/models/variant.py b/src/mavedb/models/variant.py index 59b6e729c..dd46c430c 100644 --- a/src/mavedb/models/variant.py +++ b/src/mavedb/models/variant.py @@ -9,13 +9,14 @@ if TYPE_CHECKING: from .mapped_variant import MappedVariant + from .mapping_record import MappingRecord from .score_set import ScoreSet class Variant(Base): __tablename__ = "variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), index=True, nullable=True, unique=True) data = Column(JSONB, nullable=False) @@ -35,5 +36,9 @@ class Variant(Base): back_populates="variant", cascade="all, delete-orphan" ) + mapping_records: Mapped[List["MappingRecord"]] = relationship( + back_populates="variant", cascade="all, delete-orphan" + ) + # Bidirectional relationship with ScoreCalibrationFunctionalClassification is left # purposefully undefined for performance reasons. diff --git a/src/mavedb/models/variant_annotation_status.py b/src/mavedb/models/variant_annotation_status.py index f39c47a64..357d6a6e6 100644 --- a/src/mavedb/models/variant_annotation_status.py +++ b/src/mavedb/models/variant_annotation_status.py @@ -93,7 +93,7 @@ class VariantAnnotationStatus(Base): # FK index for job_run_id — needed for CASCADE deletes on job_runs Index("ix_variant_annotation_status_job_run_id", "job_run_id"), CheckConstraint( - "annotation_type IN ('vrs_mapping', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", + "annotation_type IN ('vrs_mapping', 'cross_level_translation', 'clingen_allele_id', 'mapped_hgvs', 'variant_translation', 'gnomad_allele_frequency', 'clinvar_control', 'vep_functional_consequence', 'ldh_submission')", name="ck_variant_annotation_type_valid", ), CheckConstraint( From fb0232f5c09add1c52af6d5cec26e4fdf37a0334 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:55:21 -0700 Subject: [PATCH 08/93] chore(types): add mypy stubs for ga4gh and hgvs translation APIs Add type stubs for the VRS and hgvs APIs used by the reverse translation work so callers can be type-checked without casts. - ga4gh.core: type ga4gh_identify and re-export PrevVrsVersion - ga4gh.vrs: stub the data proxy, AlleleTranslator, and normalize; translate_from returns Any so callers annotate the concrete VRS subtype they expect without a redundant cast - hgvs.assemblymapper: stub AssemblyMapper with the g_to_t/c_to_g/c_to_p conversions used for cross-level translation --- mypy_stubs/ga4gh/core/__init__.pyi | 9 ++++++++- mypy_stubs/ga4gh/vrs/dataproxy.pyi | 8 ++++++++ mypy_stubs/ga4gh/vrs/extras/__init__.pyi | 0 mypy_stubs/ga4gh/vrs/extras/translator.pyi | 19 +++++++++++++++++++ mypy_stubs/ga4gh/vrs/normalize.pyi | 5 +++++ mypy_stubs/hgvs/assemblymapper.pyi | 20 ++++++++++++++++++++ 6 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 mypy_stubs/ga4gh/vrs/dataproxy.pyi create mode 100644 mypy_stubs/ga4gh/vrs/extras/__init__.pyi create mode 100644 mypy_stubs/ga4gh/vrs/extras/translator.pyi create mode 100644 mypy_stubs/ga4gh/vrs/normalize.pyi create mode 100644 mypy_stubs/hgvs/assemblymapper.pyi diff --git a/mypy_stubs/ga4gh/core/__init__.pyi b/mypy_stubs/ga4gh/core/__init__.pyi index f5814d7cf..8044c175f 100644 --- a/mypy_stubs/ga4gh/core/__init__.pyi +++ b/mypy_stubs/ga4gh/core/__init__.pyi @@ -1,3 +1,10 @@ from . import models as models +from .identifiers import PrevVrsVersion as PrevVrsVersion -__all__ = ["models"] +def ga4gh_identify( + vro: object, + in_place: str = ..., + as_version: PrevVrsVersion | None = ..., +) -> str | None: ... + +__all__ = ["PrevVrsVersion", "ga4gh_identify", "models"] diff --git a/mypy_stubs/ga4gh/vrs/dataproxy.pyi b/mypy_stubs/ga4gh/vrs/dataproxy.pyi new file mode 100644 index 000000000..25cabfda9 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/dataproxy.pyi @@ -0,0 +1,8 @@ +from abc import ABC + +class _DataProxy(ABC): + def get_metadata(self, identifier: str) -> dict: ... + def get_sequence(self, identifier: str, start: int | None = ..., end: int | None = ...) -> str: ... + +class SeqRepoDataProxy(_DataProxy): + def __init__(self, sr: object) -> None: ... diff --git a/mypy_stubs/ga4gh/vrs/extras/__init__.pyi b/mypy_stubs/ga4gh/vrs/extras/__init__.pyi new file mode 100644 index 000000000..e69de29bb diff --git a/mypy_stubs/ga4gh/vrs/extras/translator.pyi b/mypy_stubs/ga4gh/vrs/extras/translator.pyi new file mode 100644 index 000000000..d3bb6aeb4 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/extras/translator.pyi @@ -0,0 +1,19 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +class _Translator: + default_assembly_name: str + data_proxy: _DataProxy + identify: bool + def __init__( + self, + data_proxy: _DataProxy, + default_assembly_name: str = ..., + identify: bool = ..., + ) -> None: ... + # Returns a VRS variation (Allele/CisPhasedBlock/...); typed loosely so callers + # can annotate the concrete subtype they expect without a redundant cast. + def translate_from(self, var: str, fmt: str | None = ..., **kwargs: Any) -> Any: ... + +class AlleleTranslator(_Translator): ... diff --git a/mypy_stubs/ga4gh/vrs/normalize.pyi b/mypy_stubs/ga4gh/vrs/normalize.pyi new file mode 100644 index 000000000..1aea15b23 --- /dev/null +++ b/mypy_stubs/ga4gh/vrs/normalize.pyi @@ -0,0 +1,5 @@ +from typing import Any + +from ga4gh.vrs.dataproxy import _DataProxy + +def normalize(vo: Any, data_proxy: _DataProxy | None = ..., **kwargs: Any) -> Any: ... diff --git a/mypy_stubs/hgvs/assemblymapper.pyi b/mypy_stubs/hgvs/assemblymapper.pyi new file mode 100644 index 000000000..798878312 --- /dev/null +++ b/mypy_stubs/hgvs/assemblymapper.pyi @@ -0,0 +1,20 @@ +from typing import Any + +import hgvs.sequencevariant + +class AssemblyMapper: + def __init__( + self, + hdp: Any, + assembly_name: str = ..., + alt_aln_method: str = ..., + normalize: bool = ..., + prevalidation_level: str | None = ..., + in_par_assume: str = ..., + replace_reference: bool = ..., + *args: Any, + **kwargs: Any, + ) -> None: ... + def g_to_t(self, var_g: Any, tx_ac: str) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_g(self, var_c: Any) -> hgvs.sequencevariant.SequenceVariant: ... + def c_to_p(self, var_c: Any, translation_table: Any = ...) -> hgvs.sequencevariant.SequenceVariant: ... From bcddaddf52e3f3c723a4fa1a246e49f2e27434d5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:55:56 -0700 Subject: [PATCH 09/93] build(deps): add variant-annotation as editable sibling dependency Wire the variant-annotation package into the server extra as a local editable (PEP 660) dependency so the API can drive the reverse translation pipeline. - declare variant-annotation as a develop path dependency in pyproject.toml and the server extra; lock pulls in its transitive deps (variant-translation, biopython, openpyxl, et-xmlfile) - mount ../variant-annotation into the dev and worker containers and prepend it to PYTHONPATH so the editable install resolves - pass --no-directory to poetry install in the Dockerfile so the build does not try to install the not-yet-copied editable sibling - ignore_missing_imports for variant_annotation.* in mypy: its editable import hook is unfollowable and it ships no py.typed, so treat it as untyped rather than maintaining drift-prone local stubs - add a Makefile with dev/test/lint/format targets --- Dockerfile | 2 +- Makefile | 23 +++++++ docker-compose-dev.yml | 4 ++ poetry.lock | 133 ++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 10 +++- 5 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/Dockerfile b/Dockerfile index 422e2bf83..bc5a52c45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,7 +80,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 - COPY poetry.lock pyproject.toml ./ # installs runtime dependencies to $VIRTUAL_ENV -RUN poetry install --no-root --extras server +RUN poetry install --no-root --extras server --no-directory COPY alembic /code/alembic COPY alembic.ini /code/alembic.ini COPY src /code/src diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..16f8319bc --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +VENV := .venv/bin + +.DEFAULT_GOAL := help + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}' + +.PHONY: dev +dev: ## Install deps including local editable variant-annotation + poetry install --extras server + +.PHONY: test +test: ## Run the test suite + $(VENV)/pytest tests/ + +.PHONY: lint +lint: ## Check code with ruff + $(VENV)/ruff check src/ tests/ + +.PHONY: format +format: ## Format code with ruff + $(VENV)/ruff format src/ tests/ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9835cff4e..ac4bcff58 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -19,6 +19,7 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ports: - "8002:8000" ulimits: @@ -27,6 +28,7 @@ services: hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo worker: @@ -44,12 +46,14 @@ services: REDIS_PORT: 6379 REDIS_SSL: "false" LOG_CONFIG: dev + PYTHONPATH: "/variant-annotation:/code/src" ulimits: nofile: soft: 65536 hard: 65536 volumes: - .:/code + - ../variant-annotation:/variant-annotation - mavedb-seqrepo-dev:/usr/local/share/seqrepo depends_on: - db diff --git a/poetry.lock b/poetry.lock index 037bda653..10a9a3f08 100644 --- a/poetry.lock +++ b/poetry.lock @@ -269,6 +269,51 @@ dev = ["bandit (>=1.7,<2.0)", "build (>=0.8,<1.0)", "flake8 (>=4.0,<5.0)", "ipyt docs = ["mkdocs"] tests = ["pytest (>=7.1,<8.0)", "pytest-cov (>=4.1,<5.0)", "pytest-optional-tests", "tox (>=3.25,<4.0)", "vcrpy"] +[[package]] +name = "biopython" +version = "1.87" +description = "Freely available tools for computational molecular biology." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "biopython-1.87-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:35f13796188412f135acbed196bd6cfedc1257199d50b4883e289bbd96320efc"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63978e1b3ae040c52369bc1bd3f4c668171f5f1f0808798eccd74b575cce455d"}, + {file = "biopython-1.87-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e951f4862ffc1dccc28e1c25245059fa653d86028a88f5dfe1b7875875f3a4f"}, + {file = "biopython-1.87-cp310-cp310-win32.whl", hash = "sha256:86596b05f39afbc5984ee6239c93fd75f6a97fffb9a630d74b45652c24aab964"}, + {file = "biopython-1.87-cp310-cp310-win_amd64.whl", hash = "sha256:efc16dd8a9312eb655fa590821495b0d8ca25d19f7b4ef4fa4da9e71d59d33e5"}, + {file = "biopython-1.87-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:58e36efa7eaa8813cffe440af4824afa20cc3c21b1b179a95cc0fb15c4b83c01"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccf00d15e698656796ab14ff3eb175e282da7a08eedd36a29b6cacec5a33e97f"}, + {file = "biopython-1.87-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bab39ec4108fb2542a9f1012db8269426fd6e86ed84ebc1b0bd11134ef443dd3"}, + {file = "biopython-1.87-cp311-cp311-win32.whl", hash = "sha256:126e18ad44e959a1984560562fa4f295c075ce2e610e8ddbc9ec6f52e09f70d8"}, + {file = "biopython-1.87-cp311-cp311-win_amd64.whl", hash = "sha256:d4ff5369ffb7a966bcf921cae6c8a6eab5070b5df1c9f2ae8c18ad40fdcb7ec5"}, + {file = "biopython-1.87-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:77ccc634621904d4a8fa0a43b5e0f093fa9df8c9577ed3858af648bb3528f51e"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3428155c3e0abbed7aad5ff08e034d435b84dfe560c8ec58e7d43abda4b6a43"}, + {file = "biopython-1.87-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:65ba69ef0273e983a9036c2a228142bc34266179a5f03660fc281d332d718630"}, + {file = "biopython-1.87-cp312-cp312-win32.whl", hash = "sha256:b077777fd2c555434bdcee58743f6f860aa80e1e005d9671913aa73823c6a773"}, + {file = "biopython-1.87-cp312-cp312-win_amd64.whl", hash = "sha256:856e3d64f1f27db493474ff84916ed8572731a525e001c7d0d8f41a0fd187000"}, + {file = "biopython-1.87-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e05ef5d632c319ab3ef77705c74061190d0792b07e1f2b9eee867401b2758e7e"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:772539297fa16a78f38651c793f53f8c11bd18317b111982e72cf30a6e57512a"}, + {file = "biopython-1.87-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d221b2e08e7e89713fdbfb15c8ea6744e908d59f672cd2b6fcf9ed47910d05e"}, + {file = "biopython-1.87-cp313-cp313-win32.whl", hash = "sha256:fab1b12f6bc4646b7f56b4c390ecff685f02b5b29e3a0c10477195bb49fe62f8"}, + {file = "biopython-1.87-cp313-cp313-win_amd64.whl", hash = "sha256:01ee30203bd4b2145cdfe2878499e549a7087f897a6f4d1ebd9de30790123140"}, + {file = "biopython-1.87-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db73fe16aa2b20677ac86d1997612acb0aa1a3720bc899f65d2bce5583208e1b"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ffe272517478691439a59cccd3cc2929fc8f6bfb8cbc8cc5acc103660395a7"}, + {file = "biopython-1.87-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff28a6f31630b3c9f52903478a2ed9dd894b07c1998e40eaeefbeefac20f2d0f"}, + {file = "biopython-1.87-cp314-cp314-win32.whl", hash = "sha256:d740c75d4bc94f9dff51719a0deda37e5e885f06ee6dfbb5e9a21bbe9de35a9c"}, + {file = "biopython-1.87-cp314-cp314-win_amd64.whl", hash = "sha256:98e397096336a49804b6aaaeac8c47ad82e3e4430862f0cde37be73037f1017e"}, + {file = "biopython-1.87-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e4878a9b56775480154c686f81e98f6d907b44d87605bdc2f53538ccdfde9624"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6fcec2e3602ed52ced701f8f7851952383f84dbc4caeb4d202d088170e86b6d"}, + {file = "biopython-1.87-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8da2d44a4b912c7550a051a5ff4bb72a61decc9c4b19ea92cba4c02fffb143c"}, + {file = "biopython-1.87-cp314-cp314t-win32.whl", hash = "sha256:3670d76759c6cb53ba617f9823d3a438c1aa5415abef6addd29cb81d61d7b312"}, + {file = "biopython-1.87-cp314-cp314t-win_amd64.whl", hash = "sha256:331c4151608a1d8406eff0d3c52a0ff1fa3e82604fc85f11c696c562919fb161"}, + {file = "biopython-1.87.tar.gz", hash = "sha256:8456c803459b679a9712422e5a7fd9809f2f089bf69bb085f3b077946ac9bdbf"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "bioutils" version = "0.6.1" @@ -1432,6 +1477,19 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "eutils" version = "0.6.0" @@ -2893,6 +2951,22 @@ files = [ {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "orcid" version = "1.0.3" @@ -4796,6 +4870,61 @@ dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"] +[[package]] +name = "variant-annotation" +version = "0.1.0" +description = "Genetic variant mapping and annotation pipeline" +optional = true +python-versions = ">=3.11" +groups = ["main"] +markers = "extra == \"server\"" +files = [] +develop = true + +[package.dependencies] +biopython = "*" +boto3 = "*" +click = "*" +openpyxl = "*" +python-dotenv = "*" +redis = "*" +requests = "*" +variant-translation = "1.1.0b1" + +[package.extras] +dev = ["ruff"] +gnomad = ["hail"] +mapping = ["dcd-mapping @ git+https://github.com/VariantEffect/dcd_mapping2.git@v2026.1.2"] +publish = ["build", "twine"] +tests = ["pytest"] + +[package.source] +type = "directory" +url = "../variant-annotation" + +[[package]] +name = "variant-translation" +version = "1.1.0b1" +description = "Reverse-translate protein HGVS variants." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "variant_translation-1.1.0b1-py3-none-any.whl", hash = "sha256:cdf693ab564b767c89e7b463b1345e801d2ab4a5f312631584bf79d178ba6122"}, + {file = "variant_translation-1.1.0b1.tar.gz", hash = "sha256:5151d02ccb5c651c7ba47398d75355f2bd19a36717faa39dc2a390cfb4c7968f"}, +] + +[package.dependencies] +click = "*" +hgvs = "*" +python-dotenv = "*" +requests = "*" + +[package.extras] +dev = ["pytest", "ruff"] +publish = ["build", "twine"] + [[package]] name = "virtualenv" version = "21.3.1" @@ -5098,9 +5227,9 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "pyathena", "python-jose", "python-multipart", "requests", "slack-sdk", "starlette", "starlette-context", "uvicorn", "variant-annotation", "watchtower"] [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "64ca4755b1a5c8930882a7ad3b25feee434f42ffb02ba2841f38c86de56cc948" +content-hash = "25445affad8eb09fb30e96f152dd0f066eb6f6a4d13e61023fba5357002f62ba" diff --git a/pyproject.toml b/pyproject.toml index 7bc2556d0..427c88996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ starlette = { version = "~0.49.0", optional = true } starlette-context = { version = "^0.3.6", optional = true } slack-sdk = { version = "~3.21.3", optional = true } uvicorn = { extras = ["standard"], version = "*", optional = true } +variant-annotation = { path = "../variant-annotation", develop = true, optional = true } watchtower = { version = "~3.2.0", optional = true } asyncclick = "^8.3.0.7" filelock = "^3.29.0" @@ -93,7 +94,7 @@ SQLAlchemy = { extras = ["mypy"], version = "~2.0.0" } [tool.poetry.extras] -server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "watchtower"] +server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons", "boto3", "cdot", "cryptography", "fastapi", "hgvs", "orcid", "psycopg2", "python-jose", "python-multipart", "pyathena", "requests", "starlette", "starlette-context", "slack-sdk", "uvicorn", "variant-annotation", "watchtower"] [tool.mypy] @@ -104,6 +105,13 @@ plugins = [ ] mypy_path = "mypy_stubs" +# variant-annotation is installed as a PEP 660 editable sibling package whose import +# hook mypy cannot follow, and it ships no py.typed marker. Treat it as untyped rather +# than maintaining drift-prone local stubs for an actively-developed internal library. +[[tool.mypy.overrides]] +module = "variant_annotation.*" +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' From ec36115003b893e9496d9eb2a7f2c384c71366bd Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:57:09 -0700 Subject: [PATCH 10/93] feat(worker): write mapping job output to the allele closure tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the variant mapping job to persist its results into the new MappingRecord / Allele / MappingRecordAllele schema instead of MappedVariant, building on the closure tables for the Better Reverse Translation epic. - create one MappingRecord per variant (including failed variants, with null VRS data); successfully post-mapped variants additionally get-or-create an authoritative Allele and link it via an is_authoritative MappingRecordAllele - supersede a variant's prior live record through ValidTime supersede_with — retiring it and its allele links and inserting the new record under one timestamp — instead of flipping a current flag, so the old/new handoff is gap-free - require a TargetGeneMapping for every mapped score, raising on a miss rather than tolerating a null link, since dcd-mapping guarantees one - combine cis-phased members when deriving hgvs_assay_level - update the mapping job tests to assert against MappingRecord and the authoritative-allele link, including the gap-free supersession and cascade-retire of the prior link TODO#765: mapping is not idempotent, so each run always creates a new authoritative link. --- .../worker/jobs/variant_processing/mapping.py | 93 ++++-- .../jobs/variant_processing/test_mapping.py | 276 ++++++++++-------- 2 files changed, 229 insertions(+), 140 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index efc8226b3..d8b02cebb 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -24,12 +24,15 @@ from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User @@ -130,7 +133,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Per-(target, alignment_level) QC records produced by the dcd-mapping API. # All records share the same tool_version because they come from a single run; - # we use that as the global ``mapping_api_version`` carried on each MappedVariant. + # we use that as the global ``mapping_api_version`` carried on each MappingRecord. target_mappings_payload = mapping_results.get("target_mappings") or [] tool_version = next( (tm.get("tool_version") for tm in target_mappings_payload if tm.get("tool_version")), @@ -218,7 +221,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan continue target_gene_mapping = TargetGeneMapping( - target_gene_id=target_gene.id, + target_gene=target_gene, alignment_level=AnnotationLayer.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), @@ -268,18 +271,16 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.save_to_context({"processing_variant": variant.id}) logger.debug(f"Processing variant {variant.id}.", extra=job_manager.logging_context()) - # there should only be one current mapped variant per variant id, so update old mapped variant to current = false + # Only allow one live MappingRecord per variant. The prior live record (if any) is + # superseded by the new version below via supersede_with, which retires it (cascading to + # its allele links) and inserts the new record under one timestamp. existing_mapped_variant = ( - job_manager.db.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + job_manager.db.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - existing_mapped_variant.current = False - job_manager.db.add(existing_mapped_variant) - logger.debug(msg="Set existing mapped variant to current = false.", extra=job_manager.logging_context()) annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") if annotation_was_successful: @@ -294,27 +295,47 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan score_alignment_level = mapped_score.get("alignment_level") if not score_target or not score_alignment_level: raise NonexistentMappingResultsError( - f"ScoreAnnotation for variant {variant_urn!r} is missing " - f"target_gene_identifier or alignment_level." + f"ScoreAnnotation for variant {variant_urn!r} is missing target_gene_identifier or alignment_level." ) + pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} + post_mapped_allele: dict = mapped_score.get("post_mapped") or {} + annotation_layer = AnnotationLayer.from_wire(score_alignment_level) + assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) + + # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping + # via (target_gene_identifier, alignment_level) -- including failed variants, which + # it re-attributes to the target's preferred layer. A miss implies a malformed payload. target_gene_mapping_row = target_gene_mapping_by_key.get((score_target, score_alignment_level)) - mapped_variant = MappedVariant( - pre_mapped=mapped_score.get("pre_mapped", null()), - post_mapped=mapped_score.get("post_mapped", null()), - hgvs_assay_level=get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + if target_gene_mapping_row is None: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} has no TargetGeneMapping for " + f"(target={score_target!r}, alignment_level={score_alignment_level!r})." + ) + + mapping_record = MappingRecord( variant_id=variant.id, - modification_date=date.today(), + vrs_digest=pre_mapped_allele.get("id"), + pre_mapped=pre_mapped_allele or None, + assay_level=annotation_layer, + hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], - vrs_version=mapped_score.get("vrs_version", null()), + vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, - error_message=mapped_score.get("error_message", null()), - target_gene_mapping_id=target_gene_mapping_row.id if target_gene_mapping_row else None, - alignment_level=AnnotationLayer.from_wire(score_alignment_level), + target_gene_mapping_id=target_gene_mapping_row.id, + alignment_level=annotation_layer, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), - current=True, ) + if existing_mapped_variant: + # Retire the prior record (cascading to its allele links) and insert this one under a + # single timestamp, so the prior valid_to equals the new valid_from with no gap. + existing_mapped_variant.supersede_with(job_manager.db, mapping_record) + logger.debug( + msg="Superseded prior mapping record and its allele links.", extra=job_manager.logging_context() + ) + else: + job_manager.db.add(mapping_record) annotation_manager.add_annotation( variant_id=variant.id, # type: ignore @@ -327,14 +348,38 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan annotation_data={ "error_message": mapped_score.get("error_message", null()), "annotation_metadata": { - "mapped_assay_level_hgvs": get_hgvs_from_post_mapped(mapped_score.get("post_mapped", {})), + "mapped_assay_level_hgvs": assay_level_hgvs, }, }, current=True, ) - job_manager.db.add(mapped_variant) + # Only variants with a post-mapped representation yield an authoritative Allele. Failed variants + # get a MappingRecord (with null VRS data) but no linked allele. + if post_mapped_allele: + allele_draft = AlleleDbModel( + vrs_digest=post_mapped_allele["id"], + level=annotation_layer, + hgvs_g=assay_level_hgvs if annotation_layer == AnnotationLayer.genomic else None, + hgvs_c=assay_level_hgvs if annotation_layer == AnnotationLayer.cdna else None, + hgvs_p=assay_level_hgvs if annotation_layer == AnnotationLayer.protein else None, + post_mapped=post_mapped_allele, + ) + authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) + job_manager.db.flush() + + # TODO#765: Mapping is not idempotent, so we must always create a new link. + job_manager.db.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=authoritative_allele.id, + is_authoritative=True, + ) + ) + logger.debug(msg="Linked mapped variant to authoritative allele.", extra=job_manager.logging_context()) + logger.debug(msg="Added new mapped variant to session.", extra=job_manager.logging_context()) + job_manager.db.flush() annotation_manager.flush() diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index dc2ac843e..b28f61564 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -5,15 +5,19 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import date from unittest.mock import MagicMock, patch from sqlalchemy.exc import NoResultFound from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -24,6 +28,25 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def authoritative_allele_for(session, mapping_record): + """Return the authoritative (post-mapped) Allele linked to a mapping record, or None. + + Under the allele data model the post-mapped VRS representation lives on an + ``Allele`` joined to the ``MappingRecord`` through ``MappingRecordAllele``. + A successfully post-mapped variant has exactly one authoritative link; a + variant that failed to post-map has a mapping record but no such link. + """ + return ( + session.query(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .filter( + MappingRecordAllele.mapping_record_id == mapping_record.id, + MappingRecordAllele.is_authoritative.is_(True), + ) + .one_or_none() + ) + + @pytest.mark.unit @pytest.mark.asyncio class TestMapVariantsForScoreSetUnit: @@ -90,7 +113,10 @@ async def test_map_variants_for_score_set_no_mapped_scores( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [], "error_message": "No variants were mapped for this score set"} + { + "mapped_scores": [], + "error_message": "No variants were mapped for this score set", + } ), ), ): @@ -134,7 +160,10 @@ async def test_map_variants_for_score_set_no_reference_data( _UnixSelectorEventLoop, "run_in_executor", return_value=self.dummy_mapping_output( - {"mapped_scores": [MagicMock()], "error_message": "Reference metadata missing from mapping results"} + { + "mapped_scores": [MagicMock()], + "error_message": "Reference metadata missing from mapping results", + } ), ), ): @@ -318,9 +347,9 @@ async def dummy_mapping_job(): for target in sample_score_set.target_genes: assert target.mapped_hgnc_name is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -435,9 +464,9 @@ async def dummy_mapping_job(): else: assert target.post_mapped_metadata.get("protein") is None - # Verify that a mapped variant was created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that a mapping record was created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -501,13 +530,12 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.failed assert sample_score_set.mapping_errors["error_message"] == "All variants failed to map." - # Verify that one mapped variant was created. Although no successful mapping, an entry is still created. - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 1 + # Verify that one mapping record was created. Although no successful mapping, an entry is still created. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 1 - # Verify that the mapped variant has no post-mapped data - mapped_variant = mapped_variants[0] - assert mapped_variant.post_mapped == {} + # Verify that the mapping record has no authoritative (post-mapped) allele + assert authoritative_allele_for(session, mapping_records[0]) is None # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -584,19 +612,14 @@ async def dummy_mapping_job(): # Although only one variant was successfully mapped, verify that an entity was created # for each variant in the score set - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 - # Verify that only one variant has post-mapped data - mapped_variant_with_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped != {}).one_or_none() - ) - assert mapped_variant_with_post_data is not None - - mapped_variant_without_post_data = ( - session.query(MappedVariant).filter(MappedVariant.post_mapped == {}).one_or_none() - ) - assert mapped_variant_without_post_data is not None + # Verify that exactly one mapping record has post-mapped (authoritative) allele data + records_with_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is not None] + records_without_post_data = [r for r in mapping_records if authoritative_allele_for(session, r) is None] + assert len(records_with_post_data) == 1 + assert len(records_without_post_data) == 1 # Verify that annotation statuses were created and correct annotation_status_success = ( @@ -678,17 +701,17 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 2 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 # Verify that both variants have post-mapped data. I'm comfortable assuming the # data is correct given our layer permutation tests above. for urn in ["variant:1", "variant:2"]: - mapped_variant = session.query(MappedVariant).filter(MappedVariant.variant.has(urn=urn)).one_or_none() - assert mapped_variant is not None - assert mapped_variant.post_mapped != {} - assert mapped_variant.hgvs_assay_level is not None + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant.has(urn=urn)).one_or_none() + assert mapping_record is not None + assert authoritative_allele_for(session, mapping_record) is not None + assert mapping_record.hgvs_assay_level is not None # Verify that annotation statuses were created and correct annotation_statuses = ( @@ -733,13 +756,23 @@ async def dummy_mapping_job(): ) session.add(variant) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=AnnotationLayer.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) + session.commit() + # Link the prior record to an authoritative allele. Re-mapping must retire this link too, + # not just the record — a live link dangling off a retired record breaks the temporal model. + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=AnnotationLayer.genomic.value) + session.add(prior_allele) + session.commit() + prior_link = MappingRecordAllele( + mapping_record_id=mapping_record.id, allele_id=prior_allele.id, is_authoritative=True + ) + session.add(prior_link) session.commit() variant_annotation_status = VariantAnnotationStatus( variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" @@ -766,31 +799,42 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify the existing mapped variant was marked as non-current - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.id == mapped_variant.id, MappedVariant.current.is_(False)) + # Verify the existing mapping record was retired (valid_to closed) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.id == mapping_record.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - # Verify a new mapped variant entry was created - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + # Verify the prior record's authoritative allele link was retired alongside the record, + # so no live link dangles off the retired record. + session.refresh(prior_link) + assert prior_link.valid_to is not None + + # Verify a new, live mapping record entry was created + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None + + # Gap-free handoff: the prior record closed at exactly the new record's valid_from (one + # supersession timestamp), and the cascade closed the prior link at that same instant — so + # no point-in-time query lands in a hole between the old and new record or their links. + assert non_current_mapping_record.valid_to == new_mapping_record.valid_from + assert prior_link.valid_to == non_current_mapping_record.valid_to - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" # Verify the non-current annotation status still exists old_annotation_status = ( session.query(VariantAnnotationStatus) .filter( - VariantAnnotationStatus.variant_id == non_current_mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == non_current_mapping_record.variant_id, VariantAnnotationStatus.current.is_(False), ) .one_or_none() @@ -862,9 +906,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -875,11 +919,11 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -953,9 +997,9 @@ async def dummy_mapping_job(): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete @@ -966,11 +1010,11 @@ async def dummy_mapping_job(): assert target.mapped_hgnc_name is not None assert target.post_mapped_metadata is not None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1053,9 +1097,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1130,9 +1174,9 @@ async def dummy_mapping_job(): # Error message originates from our mock mapping construction function assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1206,9 +1250,9 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "Reference metadata missing from mapping results" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1246,20 +1290,20 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( sample_independent_variant_creation_run, ) - # Associate mapped variants with all variants just created in the score set + # Associate mapping records with all variants just created in the score set variants = session.query(Variant).filter(Variant.score_set_id == sample_score_set.id).all() for variant in variants: - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2023-01-01T00:00:00Z", + assay_level=AnnotationLayer.genomic, + mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) annotation_status = VariantAnnotationStatus( variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" ) session.add(annotation_status) - session.add(mapped_variant) + session.add(mapping_record) session.commit() async def dummy_mapping_job(): @@ -1295,27 +1339,27 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that mapped variants were marked as non-current and new entries created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == len(variants) * 2 # Each variant has two mapped entries now + # Verify that mapping records were marked as non-current and new entries created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == len(variants) * 2 # Each variant has two mapping records now for variant in variants: - non_current_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(False)) + non_current_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.valid_to.isnot(None)) .one_or_none() ) - assert non_current_mapped_variant is not None + assert non_current_mapping_record is not None - new_mapped_variant = ( - session.query(MappedVariant) - .filter(MappedVariant.variant_id == variant.id, MappedVariant.current.is_(True)) + new_mapping_record = ( + session.query(MappingRecord) + .filter(MappingRecord.variant_id == variant.id, MappingRecord.current) .one_or_none() ) - assert new_mapped_variant is not None + assert new_mapping_record is not None - # Verify that the new mapped variant has updated mapping data - assert new_mapped_variant.mapped_date != "2023-01-01T00:00:00Z" - assert new_mapped_variant.mapping_api_version != "v1.0.0" + # Verify that the new mapping record has updated mapping data + assert new_mapping_record.mapped_date != date(2023, 1, 1) + assert new_mapping_record.mapping_api_version != "v1.0.0" # Verify that annotation statuses where marked as non-current and new entries created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1389,9 +1433,9 @@ async def dummy_mapping_job(): assert sample_score_set.mapping_errors is not None assert "test error: no mapped scores" in sample_score_set.mapping_errors["error_message"] - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1448,9 +1492,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1518,19 +1562,19 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1606,19 +1650,19 @@ async def dummy_mapping_job(): await arq_worker.async_run() await arq_worker.run_check() - # Verify that mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 4 + # Verify that mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 4 # Verify score set mapping state assert sample_score_set.mapping_state == MappingState.complete assert sample_score_set.mapping_errors is None - # Verify that each variant has a corresponding mapped variant + # Verify that each variant has a corresponding mapping record variants = ( session.query(Variant) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, MappedVariant.current.is_(True)) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, MappingRecord.current.is_(True)) .all() ) assert len(variants) == 4 @@ -1690,9 +1734,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -1744,9 +1788,9 @@ async def dummy_mapping_job(): in sample_score_set.mapping_errors["error_message"] ) - # Verify that no mapped variants were created - mapped_variants = session.query(MappedVariant).all() - assert len(mapped_variants) == 0 + # Verify that no mapping records were created + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 0 # Verify that no annotation statuses were created annotation_statuses = session.query(VariantAnnotationStatus).all() From b70611f5028089884b4771a174cec5fd968fef82 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:57:29 -0700 Subject: [PATCH 11/93] fixup into cisphased hgvs commit --- tests/lib/test_variants.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index ca9c2b0b3..9cffaa793 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -46,6 +46,18 @@ def test_get_hgvs_from_post_mapped_cis_phased_block(): assert result is None +def test_get_hgvs_from_post_mapped_cis_phased_block_combine_cis(): + # combine_cis collapses the cis-phased members into one bracketed expression. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, combine_cis=True) + assert result == "NM_003345:p.[Asp5Phe;Asp5Phe]" + + +def test_get_hgvs_from_post_mapped_single_allele_combine_cis_is_unbracketed(): + # A single-variant post-mapped allele is unaffected by combine_cis. + result = get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, combine_cis=True) + assert result == TEST_HGVS_IDENTIFIER + + def test_get_hgvs_from_post_mapped_single_allele_vrs_1(): with pytest.raises(ValueError): get_hgvs_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X) From c6a06e665076d56e6d1dc23665ea7365bfd0f663 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:58:35 -0700 Subject: [PATCH 12/93] feat(worker): provide a SeqRepo data proxy in the worker context Add a seqrepo_data_proxy to the worker context in both the standalone context and the on_job_start hook, so jobs performing VRS translation have a SeqRepo-backed data proxy available alongside the cdot HGVS data provider. Mirror it in the mock worker context used by tests. --- src/mavedb/worker/settings/lifecycle.py | 4 +++- tests/worker/conftest_optional.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mavedb/worker/settings/lifecycle.py b/src/mavedb/worker/settings/lifecycle.py index 54a0b4c76..5be66bd2d 100644 --- a/src/mavedb/worker/settings/lifecycle.py +++ b/src/mavedb/worker/settings/lifecycle.py @@ -9,7 +9,7 @@ from concurrent import futures -from mavedb.data_providers.services import cdot_rest +from mavedb.data_providers.services import cdot_rest, seqrepo_data_proxy def standalone_ctx(): @@ -18,6 +18,7 @@ def standalone_ctx(): ctx["pool"] = futures.ProcessPoolExecutor() ctx["redis"] = None # Redis connection can be set up here if needed. ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} # Additional context setup can be added here as needed. @@ -37,6 +38,7 @@ async def shutdown(ctx): async def on_job_start(ctx): ctx["hdp"] = cdot_rest() + ctx["seqrepo"] = seqrepo_data_proxy() ctx["state"] = {} diff --git a/tests/worker/conftest_optional.py b/tests/worker/conftest_optional.py index 0f1d2e95f..ccd4208a0 100644 --- a/tests/worker/conftest_optional.py +++ b/tests/worker/conftest_optional.py @@ -4,6 +4,7 @@ import pytest from arq import ArqRedis from cdot.hgvs.dataproviders import RESTDataProvider +from ga4gh.vrs.dataproxy import SeqRepoDataProxy from sqlalchemy.orm import Session from mavedb.worker.lib.managers.job_manager import JobManager @@ -52,6 +53,7 @@ def mock_worker_ctx(): """Create a mock worker context dictionary for testing.""" mock_redis = Mock(spec=ArqRedis) mock_hdp = Mock(spec=RESTDataProvider) + mock_seqrepo = Mock(spec=SeqRepoDataProxy) mock_pool = Mock(spec=ProcessPoolExecutor) # Don't mock the session itself to allow real DB interactions in tests @@ -60,5 +62,6 @@ def mock_worker_ctx(): return { "redis": mock_redis, "hdp": mock_hdp, + "seqrepo": mock_seqrepo, "pool": mock_pool, } From a8e16363080b87a284bdbfa3ac3e81d21f28bdf5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:58:41 -0700 Subject: [PATCH 13/93] fixup into mapper changes --- tests/helpers/util/setup/worker.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index cb6949b4e..815dd1a90 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -4,13 +4,13 @@ from sqlalchemy import select +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant from mavedb.worker.jobs import ( create_variants_for_score_set, map_variants_for_score_set, ) -from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import ( TEST_CODING_LAYER, @@ -23,6 +23,16 @@ ) +# Placeholder reference accessions for qualifying target-relative HGVS in mock +# post-mapped output, keyed by the HGVS kind prefix (``c.``/``n.``/``g.``/``p.``). +_PLACEHOLDER_ACCESSIONS = { + "c.": "NM_999999.1", + "n.": "NR_999999.1", + "g.": "NC_999999.1", + "p.": "NP_999999.1", +} + + async def create_variants_in_score_set( session, mock_s3_client, score_df, count_df, mock_worker_ctx, variant_creation_run ): @@ -128,6 +138,10 @@ async def construct_mock_mapping_output( "target_gene_identifier": target.name, "alignment_level": layer, "preferred": idx == 0, + # The accession this level was aligned against — a transcript (NM_) + # only at the coding level. Reverse translation resolves its coding + # transcript hint from the cdna entry's reference_accession. + "reference_accession": _PLACEHOLDER_ACCESSIONS[f"{layer}."], "tool_name": "dcd-mapping", "tool_version": "pytest.0.0", "tool_parameters": {}, @@ -155,10 +169,15 @@ async def construct_mock_mapping_output( "alignment_level": default_alignment_level, } - # Don't alter HGVS strings in post mapped output. This makes it considerably - # easier to assert correctness in tests. + # Reuse the variant's own HGVS in post-mapped output to keep assertions simple. + # Real mapper post-mapped HGVS is always accession-qualified (it is mapped onto + # a reference sequence), so prefix a placeholder accession when the source HGVS + # is target-relative (a bare ``c.``/``n.``/``g.``/``p.`` with no accession). if with_post_mapped: - mapped_score["post_mapped"]["expressions"][0]["value"] = variant.hgvs_nt or variant.hgvs_pro + hgvs = variant.hgvs_nt or variant.hgvs_pro + if hgvs and ":" not in hgvs: + hgvs = f"{_PLACEHOLDER_ACCESSIONS.get(hgvs[:2], 'NM_999999.1')}:{hgvs}" + mapped_score["post_mapped"]["expressions"][0]["value"] = hgvs # Skip every other variant if not with_all_variants if not with_all_variants and idx % 2 == 0: From 3bd97176ea7d96e0c607c15a27a5398afcfc0858 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:59:45 -0700 Subject: [PATCH 14/93] feat(worker): add reverse translation job for cross-level HGVS alleles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a reverse_translate_variants_for_score_set worker job that builds the cross-level HGVS equivalence class for every mapped variant in a score set, persisting the candidates as non-authoritative alleles. - for each current authoritative MappingRecord, collapse the assay-level HGVS to its protein consequence and expand to all coding/genomic candidates via a single batched construct_equivalent_variants call from the variant-annotation library, run off-loop in a thread - resolve each record's coding (NM_) transcript from its target gene's cdna alignment, falling back to a batched UTA NP_→NM_ lookup for protein-level mappings; records with no coding transcript are skipped - translate each candidate to VRS and write it as a get-or-created Allele linked non-authoritatively to the record, deduping by vrs_digest and never relinking the record's authoritative allele - supersede the prior live derived links with the new set in one gap-free operation via ValidTime.supersede_live_where - record per-variant cross_level_translation annotation status (success / failed / skipped), retaining per-candidate translation errors as metadata; the job fails only when every variant fails - add WorkerCoordinateTranslator and NullTranscriptSource adapters for the library's translation ports, deferring AssemblyMapper init so its network calls do not fire under mocked unit tests - get_or_create_allele helper, job registration, and a pipeline dependency placing reverse translation after mapping and before CAR submission TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the records; idempotent records would let unchanged derived links stay live. --- src/mavedb/lib/variant_translations.py | 20 + src/mavedb/lib/workflow/definitions.py | 12 +- src/mavedb/worker/jobs/__init__.py | 2 + src/mavedb/worker/jobs/registry.py | 9 + .../jobs/variant_processing/__init__.py | 2 + .../variant_processing/reverse_translation.py | 477 +++++++++ src/mavedb/worker/lib/translation_ports.py | 59 ++ tests/worker/jobs/conftest.py | 32 + .../test_reverse_translation.py | 919 ++++++++++++++++++ 9 files changed, 1531 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/worker/jobs/variant_processing/reverse_translation.py create mode 100644 src/mavedb/worker/lib/translation_ports.py create mode 100644 tests/worker/jobs/variant_processing/test_reverse_translation.py diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index c24c5ed95..701cc17d1 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -7,10 +7,12 @@ from typing import cast +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import CursorResult from sqlalchemy.orm import Session +from mavedb.models.allele import Allele from mavedb.models.variant_translation import VariantTranslation @@ -39,3 +41,21 @@ def upsert_variant_translations(db: Session, translations: list[tuple[str, str]] created = result.rowcount existing = len(unique) - created return created, existing + + +def get_or_create_allele(db: Session, allele_draft: Allele) -> Allele: + """Return the existing Allele matching ``allele_draft``'s vrs_digest, else add the draft. + + This is a get-or-create, not an upsert: a matching row is returned untouched, and on + a miss the draft is added to the session (not flushed). The draft is never used to + update an existing row. + + NOTE: This function does not persist the returned Allele to the database; the caller + is responsible for committing the session. + """ + existing = db.scalars(select(Allele).where(Allele.vrs_digest == allele_draft.vrs_digest)).one_or_none() + if existing is not None: + return existing + + db.add(allele_draft) + return allele_draft diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 056ff5309..3d3b70511 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -20,6 +20,16 @@ def annotation_pipeline_job_definitions( [(upstream_mapping_key, DependencyType.SUCCESS_REQUIRED)] if upstream_mapping_key else [] ) return [ + { + "key": "reverse_translate_variants_for_score_set", + "function": "reverse_translate_variants_for_score_set", + "type": JobType.MAPPED_VARIANT_ANNOTATION, + "params": { + "correlation_id": None, # Required param to be filled in at runtime + "score_set_id": None, # Required param to be filled in at runtime + }, + "dependencies": mapping_dep, + }, { "key": "submit_score_set_mappings_to_car", "function": "submit_score_set_mappings_to_car", @@ -29,7 +39,7 @@ def annotation_pipeline_job_definitions( "score_set_id": None, # Required param to be filled in at runtime "updater_id": None, # Required param to be filled in at runtime }, - "dependencies": mapping_dep, + "dependencies": [("reverse_translate_variants_for_score_set", DependencyType.SUCCESS_REQUIRED)], }, { "key": "warm_clingen_cache", diff --git a/src/mavedb/worker/jobs/__init__.py b/src/mavedb/worker/jobs/__init__.py index e421bbad2..488398117 100644 --- a/src/mavedb/worker/jobs/__init__.py +++ b/src/mavedb/worker/jobs/__init__.py @@ -33,11 +33,13 @@ from mavedb.worker.jobs.variant_processing.mapping import ( map_variants_for_score_set, ) +from mavedb.worker.jobs.variant_processing.reverse_translation import reverse_translate_variants_for_score_set __all__ = [ # Variant processing jobs "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", # External service integration jobs "submit_score_set_mappings_to_car", "submit_score_set_mappings_to_ldh", diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index ba9754a05..c445098ef 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -32,6 +32,7 @@ from mavedb.worker.jobs.variant_processing import ( create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, ) # All job functions for ARQ worker @@ -39,6 +40,7 @@ # Variant processing jobs create_variants_for_score_set, map_variants_for_score_set, + reverse_translate_variants_for_score_set, # External service jobs submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -102,6 +104,13 @@ "key": "map_variants_for_score_set", "type": JobType.VARIANT_MAPPING, }, + reverse_translate_variants_for_score_set: { + "dependencies": [], + "params": {"score_set_id": None, "correlation_id": None}, + "function": "reverse_translate_variants_for_score_set", + "key": "reverse_translate_variants_for_score_set", + "type": JobType.VARIANT_MAPPING, + }, submit_score_set_mappings_to_car: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/src/mavedb/worker/jobs/variant_processing/__init__.py b/src/mavedb/worker/jobs/variant_processing/__init__.py index a6df09753..240de5e28 100644 --- a/src/mavedb/worker/jobs/variant_processing/__init__.py +++ b/src/mavedb/worker/jobs/variant_processing/__init__.py @@ -10,8 +10,10 @@ from .mapping import ( map_variants_for_score_set, ) +from .reverse_translation import reverse_translate_variants_for_score_set __all__ = [ "create_variants_for_score_set", "map_variants_for_score_set", + "reverse_translate_variants_for_score_set", ] diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py new file mode 100644 index 000000000..61ca1d883 --- /dev/null +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -0,0 +1,477 @@ +"""Reverse translation worker job — builds cross-level HGVS equivalence classes. + +For each mapped variant in a score set, calls construct_equivalent_variants from +the variant-annotation library to produce all coding and genomic HGVS candidates +encoding the same protein consequence. The candidates are written as non-authoritative +Allele rows linked to the existing MappingRecord via MappingRecordAllele. +""" + +import asyncio +import contextlib +import dataclasses +import functools +import logging +import os +from typing import Any, NamedTuple, Sequence + +from ga4gh.vrs.extras.translator import AlleleTranslator +from sqlalchemy import select +from variant_annotation.lib.accessions import looks_like_refseq_protein_accession +from variant_annotation.lib.clients.uta import UtaClient, connect_uta +from variant_annotation.lib.translation import construct_equivalent_variants +from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode + +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.hgvs import extract_accession +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.variant_translations import get_or_create_allele +from mavedb.lib.vrs_utils import translate_hgvs_to_variation +from mavedb.models.allele import Allele as AlleleDbModel +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping +from mavedb.models.variant import Variant +from mavedb.worker.jobs.utils.setup import validate_job_params +from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management +from mavedb.worker.lib.managers.job_manager import JobManager +from mavedb.worker.lib.translation_ports import NullTranscriptSource, WorkerCoordinateTranslator + +logger = logging.getLogger(__name__) + +# Job defaults when no `translation_config` override is supplied: enumerate the full coding +# equivalence class — every synonymous codon, indels included. wt_codon_mode "all" requires +# include_indels=True (TranslationConfig enforces this). +_DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { + "include_indels": True, + "wt_codon_mode": WtCodonMode.ALL, +} + + +class _TranscriptResolution(NamedTuple): + """A mapping record paired with what we know about its coding transcript before the + batched UTA lookup: the gene-level cdna transcript if the mapper supplied one, and the + protein accession still awaiting NP_→NM_ resolution otherwise. Exactly one of the two is + set for a resolvable record; both are None when neither path applies (record is skipped).""" + + rec: MappingRecord + variant: Variant + gene_transcript: str | None + protein_accession: str | None + + +def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: + """Resolve each RefSeq protein accession (NP_/XP_…) to its preferred coding transcript + via UTA's associated_accessions table. + + Protein-level mappings carry no coding transcript — the mapper emits reference_accession + only for cdna alignments — so reverse-translating them relies on the NP_→NM_ association + UTA records. Resolution and the multi-transcript preference order both live in the + variant-annotation library's UtaClient; we own only the connection lifecycle here so the + long-lived worker process does not leak UTA connections across jobs. + """ + if not protein_accessions: + return {} + + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve protein→transcript associations.") + + with contextlib.closing(connect_uta(uta_db_url)) as conn: + client = UtaClient(conn) + return { + pro_ac: transcript + for pro_ac in sorted(protein_accessions) + if (transcript := client.transcript_for_protein(pro_ac)) is not None + } + + +def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: + """Build the variant-annotation TranslationConfig from optional job-param overrides. + + Each key in `overrides` is a TranslationConfig field (that dataclass is the source of truth + for the available knobs) and wins over the job defaults in _DEFAULT_TRANSLATION_CONFIG; + absent or None means use the defaults. wt_codon_mode is coerced to the enum so a JSON string + value ("all"/"unambiguous"/"none") works. + + Raises ValueError with an actionable message — listing the offending value and the valid + options — for an unknown field, an invalid wt_codon_mode, or an invalid combination (e.g. a + wt_codon_mode other than "none" without include_indels). The allowed-field set is derived + from TranslationConfig itself, so it never drifts from the library. + """ + config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} + + allowed_fields = {field.name for field in dataclasses.fields(TranslationConfig)} + unknown_fields = set(config_kwargs) - allowed_fields + if unknown_fields: + raise ValueError( + f"Unknown translation_config option(s): {', '.join(sorted(unknown_fields))}. " + f"Valid options: {', '.join(sorted(allowed_fields))}." + ) + + raw_mode = config_kwargs.get("wt_codon_mode") + if raw_mode is not None: + try: + config_kwargs["wt_codon_mode"] = WtCodonMode(raw_mode) + except ValueError: + valid_modes = ", ".join(repr(mode.value) for mode in WtCodonMode) + raise ValueError( + f"Invalid translation_config wt_codon_mode {raw_mode!r}. Valid values: {valid_modes}." + ) from None + + try: + return TranslationConfig(**config_kwargs) + except ValueError as exc: + raise ValueError(f"Invalid translation_config: {exc}") from exc + + +@with_pipeline_management +async def reverse_translate_variants_for_score_set( + ctx: dict, job_id: int, job_manager: JobManager +) -> JobExecutionOutcome: + """Build the cross-level HGVS equivalence class for every mapped variant in the score set. + + Reads current MappingRecords that carry an hgvs_assay_level string, collapses each + to its ProteinConsequence, and expands to all coding/genomic HGVS candidates via a + single batched subprocess call to the variant-annotation library. Each candidate is + written as a non-authoritative Allele linked to the MappingRecord. + + Required job_params: + - score_set_id (int): ID of the ScoreSet to process + - correlation_id (str): Correlation ID for tracking + + Optional job_params: + - translation_config (dict): Overrides for any variant-annotation TranslationConfig + field (e.g. include_indels, wt_codon_mode, max_indel_size). Omitted keys fall back to + the job defaults in _DEFAULT_TRANSLATION_CONFIG (full codon equivalence class, indels + included). + """ + job = job_manager.get_job() + validate_job_params(["score_set_id", "correlation_id"], job) + + score_set_id: int = job.job_params["score_set_id"] # type: ignore[index] + correlation_id: str = job.job_params["correlation_id"] # type: ignore[index] + translation_config = _build_translation_config(job.job_params.get("translation_config")) # type: ignore[union-attr] + score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one() + + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "reverse_translate_variants_for_score_set", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) + job_manager.update_progress(0, 100, "Starting reverse translation job.") + logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) + + # Build {target_gene_id -> NM_ transcript} from the cdna TargetGeneMappings the mapper + # emits. Reverse translation must run against the cdna (NM_) transcript. + cdna_transcript_by_gene: dict[int, str | None] = dict( + job_manager.db.execute( + select(TargetGeneMapping.target_gene_id, TargetGeneMapping.reference_accession) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where(TargetGene.score_set_id == score_set_id) + .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) + .where(TargetGeneMapping.reference_accession.isnot(None)) + ) + .tuples() + .all() + ) + + # Load current, authoritative, and successfully-mapped MappingRecords along with their + # target gene (for the coding-transcript lookup) and parent Variant. + rows: Sequence[tuple[MappingRecord, Variant, int]] = ( + job_manager.db.execute( + select(MappingRecord, Variant, TargetGeneMapping.target_gene_id) + .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + ) + .tuples() + .all() + ) + + if not rows: + logger.warning( + msg="No current and authoritative mapping records found for this score set.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + + # Resolve each record's coding transcript. Genomic/cdna mappings share their target + # gene's cdna alignment reference; protein mappings have none (the mapper emits + # reference_accession only for cdna alignments), so gather their protein accessions and + # resolve NP_→NM_ from UTA in a single batch query below. + transcript_resolutions: list[_TranscriptResolution] = [] + protein_accessions: set[str] = set() + for rec, variant, target_gene_id in rows: + coding_accession = cdna_transcript_by_gene.get(target_gene_id) + protein_accession = None + if not coding_accession and rec.hgvs_assay_level is not None: + raw_accession = extract_accession(rec.hgvs_assay_level) + if looks_like_refseq_protein_accession(raw_accession): + protein_accession = raw_accession + protein_accessions.add(raw_accession) + + transcript_resolutions.append(_TranscriptResolution(rec, variant, coding_accession, protein_accession)) + + transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + + # Build VariantInputs, supplying the resolved coding transcript for every input + # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence + # anchored on that transcript). Object identity on each VariantInput is preserved + # through the call so we can correlate TranslationResult.input back to its MappingRecord. + # + # A record with no coding transcript (e.g. a regulatory element aligned only at the + # genomic level, or a protein with no UTA association) has no protein consequence to + # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. + variant_inputs: list[Any] = [] + variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} + skipped_variants: list[tuple[MappingRecord, Variant]] = [] + for p in transcript_resolutions: + transcript = p.gene_transcript or ( + transcript_by_protein.get(p.protein_accession) if p.protein_accession else None + ) + if not transcript or not p.rec.hgvs_assay_level: + skipped_variants.append((p.rec, p.variant)) + continue + + inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) + variant_inputs.append(inp) + variant_input_map[id(inp)] = (p.rec, p.variant) + + total = len(variant_inputs) + job_manager.save_to_context({"total_variants": total, "skipped_variants": len(skipped_variants)}) + job_manager.update_progress( + 10, + 100, + f"Prepared {total} variants for reverse translation ({len(skipped_variants)} skipped, no coding transcript).", + ) + logger.info( + msg=f"Running reverse translation for {total} variants ({len(skipped_variants)} skipped).", + extra=job_manager.logging_context(), + ) + + # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the + # default thread pool rather than the process pool to avoid pickling the port objects. + coordinates = WorkerCoordinateTranslator(ctx["hdp"]) + transcripts = NullTranscriptSource() + loop = asyncio.get_running_loop() + job_manager.update_progress(20, 100, "Running reverse translation subprocess.") + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) + + job_manager.update_progress(70, 100, "Writing translated alleles to database.") + logger.info( + msg=f"Translation complete: {len(results)} succeeded, {len(errors)} failed.", + extra=job_manager.logging_context(), + ) + + translated = 0 + failed = 0 + alleles_created = 0 + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + allele_translator = AlleleTranslator(ctx["seqrepo"]) + + current_record_ids = ( + select(MappingRecord.id) + .join(Variant, MappingRecord.variant_id == Variant.id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current.is_(True)) + ) + + # Collect new MappingRecord -> Allele links and defer linkage until all candidates are processed. + # This allows us to supersede the prior live derived links with the new set in one atomic operation at the end, + # so the retire and insert share a timestamp and there is no gap between the old and new sets of live links. + new_links: list[MappingRecordAllele] = [] + + # The live authoritative (record, allele) pairs. A derived candidate that equals a record's + # authoritative allele is not linked again. + authoritative_pairs: set[tuple[int, int]] = { + (record_id, allele_id) + for record_id, allele_id in job_manager.db.execute( + select(MappingRecordAllele.mapping_record_id, MappingRecordAllele.allele_id) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) + ) + .tuples() + .all() + if record_id is not None and allele_id is not None + } + + for result in results: + rec, variant = variant_input_map[id(result.input)] + candidate_count = 0 + + # Equivalence generation may surface the same VRS object more than once. + # Dedup by vrs_digest per mapping record to avoid duplicate links. + seen_digests: set[str] = set() + failed_candidates: list[dict[str, str]] = [] + candidates: list[tuple[str, AnnotationLayer, str]] = [ + (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates + ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] + + for hgvs, level, hgvs_field in candidates: + # A candidate the equivalence class produced may be a form ga4gh cannot + # translate (an intronic projection, an unsupported edge case, a malformed + # bracketed expression). A variant with at least one translatable candidate still + # advances the pipeline; only a variant where every candidate fails is + # recorded as failed (with the per-candidate errors retained as metadata). + try: + variation = translate_hgvs_to_variation(hgvs, allele_translator) + except Exception as e: + logger.warning( + msg=f"Failed to translate candidate HGVS to VRS: {hgvs} ({e})", + extra=job_manager.logging_context(), + ) + failed_candidates.append({"hgvs": hgvs, "level": level.value, "error": str(e)}) + continue + + if variation.id in seen_digests: + continue + + seen_digests.add(variation.id) + draft_allele = AlleleDbModel( + vrs_digest=variation.id, + post_mapped=variation.model_dump(), + level=level, + **{hgvs_field: hgvs}, # type: ignore[arg-type] + ) + allele = get_or_create_allele(job_manager.db, draft_allele) + job_manager.db.flush() + + if (rec.id, allele.id) in authoritative_pairs: + continue # already linked + + new_links.append( + MappingRecordAllele( + mapping_record_id=rec.id, + allele_id=allele.id, + is_authoritative=False, + ) + ) + candidate_count += 1 + + alleles_created += candidate_count + annotation_metadata = { + "hgvs_input": result.input.hgvs, + "hgvs_c_candidates": result.hgvs_c_candidates, + "hgvs_g_candidates": result.hgvs_g_candidates, + "alleles_created": candidate_count, + "failed_candidates": failed_candidates, + } + + # No translatable candidates and failures mean the variant failed reverse translation. No + # failures and no candidates is a success with no alleles created. + if candidate_count == 0 and failed_candidates: + failed += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.UNKNOWN, + annotation_data={ + "error_message": "All candidate HGVS failed VRS translation.", + "annotation_metadata": annotation_metadata, + }, + ) + else: + translated += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.SUCCESS, + annotation_data={"annotation_metadata": annotation_metadata}, + ) + + # Supersede the prior live derived links with the new set in one gap-free operation. + # TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the + # records, so we cannot tell which derivations are unchanged; idempotent mapping records would let + # unchanged derived links stay live instead of being retired and recreated. + MappingRecordAllele.supersede_live_where( + job_manager.db, + new_links, + MappingRecordAllele.is_authoritative.is_(False), + MappingRecordAllele.mapping_record_id.in_(current_record_ids), + ) + + for error in errors: + _rec, variant = variant_input_map[id(error.input)] + failed += 1 + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.UNKNOWN, + annotation_data={ + "error_message": error.error, + "annotation_metadata": {"hgvs_input": error.input.hgvs}, + }, + ) + + skipped = len(skipped_variants) + for rec, variant in skipped_variants: + annotation_manager.add_annotation( + variant_id=variant.id, + annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, + status=AnnotationStatus.SKIPPED, + annotation_data={ + "annotation_metadata": { + "hgvs_input": rec.hgvs_assay_level, + "reason": "No coding transcript for target gene; no protein consequence to reverse-translate.", + } + }, + ) + + annotation_manager.flush() + + job_manager.save_to_context( + { + "translated": translated, + "failed": failed, + "skipped": skipped, + "alleles_created": alleles_created, + } + ) + logger.info( + msg=( + f"Reverse translation complete: {translated} translated, {failed} failed, " + f"{skipped} skipped, {alleles_created} alleles created." + ), + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + + if translated == 0 and failed > 0: + logger.error( + msg="All variant reverse translations failed.", + extra=job_manager.logging_context(), + ) + return JobExecutionOutcome.failed( + reason="All variant reverse translations failed.", + data={"translated": 0, "failed": failed, "skipped": skipped, "alleles_created": 0}, + failure_category=FailureCategory.DATA_ERROR, + ) + + return JobExecutionOutcome.succeeded( + data={"translated": translated, "failed": failed, "skipped": skipped, "alleles_created": alleles_created} + ) diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py new file mode 100644 index 000000000..467d2f93a --- /dev/null +++ b/src/mavedb/worker/lib/translation_ports.py @@ -0,0 +1,59 @@ +"""Worker-side adapters for the variant_annotation translation ports. + +construct_equivalent_variants depends on two protocols — CoordinateTranslator +and TranscriptSource. These are the worker's concrete implementations, so jobs +can pass them in without importing the port definitions directly. + +WorkerCoordinateTranslator defers AssemblyMapper initialization until the first +call — the hgvs library makes network calls on mapper construction that must not +fire in unit-test contexts where construct_equivalent_variants is mocked. +""" + +from __future__ import annotations + +from typing import Any + + +class WorkerCoordinateTranslator: + """CoordinateTranslator backed by the worker's HGVS data provider.""" + + def __init__(self, hdp: Any) -> None: + self._hdp = hdp + self._parser: Any = None + self._mapper: Any = None + + def _ensure_initialized(self) -> None: + if self._mapper is None: + import hgvs.assemblymapper + import hgvs.parser + + self._parser = hgvs.parser.Parser() + self._mapper = hgvs.assemblymapper.AssemblyMapper( + self._hdp, + assembly_name="GRCh38", + alt_aln_method="splign", + ) + + def c_to_p(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_p(self._parser.parse(c_hgvs))) + + def g_to_c(self, g_hgvs: str, transcript: str) -> str: + self._ensure_initialized() + return str(self._mapper.g_to_t(self._parser.parse(g_hgvs), transcript)) + + def c_to_g(self, c_hgvs: str) -> str: + self._ensure_initialized() + return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) + + +class NullTranscriptSource: + """Null TranscriptSource — the reverse-translation job resolves every coding transcript + itself (including the UTA NP_→NM_ lookup) and always supplies it via VariantInput.transcript, + so the library never needs to resolve one.""" + + def transcript_for_protein(self, protein_accession: str) -> str | None: + return None + + def codon_at(self, transcript: str, aa_position: int) -> str | None: + return None diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index de835a56d..eac380862 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -44,6 +44,38 @@ def map_variants_sample_params(with_populated_domain_data, sample_score_set, sam } +@pytest.fixture +def reverse_translate_variants_sample_params(with_populated_domain_data, sample_score_set): + """Provide sample parameters for reverse_translate_variants_for_score_set job.""" + + return { + "score_set_id": sample_score_set.id, + "correlation_id": "sample-reverse-translation-correlation-id", + } + + +@pytest.fixture +def sample_independent_reverse_translation_run(reverse_translate_variants_sample_params): + """Create a JobRun instance for the reverse_translate_variants_for_score_set job.""" + + return JobRun( + urn="test:reverse_translate_variants_for_score_set", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=reverse_translate_variants_sample_params, + ) + + +@pytest.fixture +def with_reverse_translation_run(session, sample_independent_reverse_translation_run): + """Add a reverse_translate_variants_for_score_set job run to the session.""" + + session.add(sample_independent_reverse_translation_run) + session.commit() + + @pytest.fixture def link_gnomad_variants_sample_params(with_populated_domain_data, sample_score_set): """Provide sample parameters for create_variants_for_score_set job.""" diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py new file mode 100644 index 000000000..c7e434845 --- /dev/null +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -0,0 +1,919 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from asyncio.unix_events import _UnixSelectorEventLoop +from unittest.mock import patch + +from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode + +from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.job_run import JobRun +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set +from mavedb.worker.jobs.variant_processing.reverse_translation import ( + _build_translation_config, + reverse_translate_variants_for_score_set, +) +from mavedb.worker.lib.managers.job_manager import JobManager +from tests.helpers.constants import TEST_GA4GH_IDENTIFIER +from tests.helpers.util.setup.worker import construct_mock_mapping_output + +pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") + +# Module under test, used as the patch root for the external annotation-package calls. +RT_MODULE = "mavedb.worker.jobs.variant_processing.reverse_translation" + + +class FakeVrsVariation: + """Stand-in for a ga4gh.vrs variation returned by translate_hgvs_to_variation. + + The reverse translation job only reads ``.id`` (for the VRS digest / dedup key) + and ``.model_dump()`` (persisted into the ``post_mapped`` JSONB column), so the + fake exposes exactly those. ``vrs_type`` lets a test assert whether a candidate + became a single Allele or a CisPhasedBlock. + """ + + def __init__(self, vrs_id: str, vrs_type: str = "Allele"): + self.id = vrs_id + self.vrs_type = vrs_type + + def model_dump(self) -> dict: + return {"type": self.vrs_type, "id": self.id} + + +def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``construct_equivalent_variants``. + + The job correlates each TranslationResult/TranslationError back to its originating + mapping record via object identity on ``result.input``, so the fake must echo the + exact VariantInput instances it was handed. Candidates are keyed by the input's + assay-level HGVS string. + """ + errors_by_hgvs = errors_by_hgvs or {} + + def _construct(inputs, *, transcripts, coordinates, config): + results, errors = [], [] + for inp in inputs: + if inp.hgvs in errors_by_hgvs: + errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) + elif inp.hgvs in results_by_hgvs: + c_candidates, g_candidates = results_by_hgvs[inp.hgvs] + results.append( + TranslationResult( + input=inp, + hgvs_c_candidates=list(c_candidates), + hgvs_g_candidates=list(g_candidates), + ) + ) + else: + errors.append(TranslationError(input=inp, error=f"no stub result for {inp.hgvs!r}")) + return results, errors + + return _construct + + +def fake_translate(id_by_hgvs: dict, type_by_hgvs: dict | None = None, errors_by_hgvs: dict | None = None): + """Build a stand-in for ``translate_hgvs_to_variation`` mapping candidate HGVS -> VRS id. + + Unknown HGVS get a deterministic id derived from the string so distinct candidates + stay distinct; supplying explicit ids lets a test force two candidates to collapse. + ``type_by_hgvs`` lets a test mark a bracketed candidate as a ``CisPhasedBlock`` (the + real translate_hgvs_to_variation returns one for cis-phased multivariants). + ``errors_by_hgvs`` makes a candidate raise, standing in for an HGVS form ga4gh cannot + translate to VRS. + """ + type_by_hgvs = type_by_hgvs or {} + errors_by_hgvs = errors_by_hgvs or {} + + def _translate(hgvs: str, translator=None) -> FakeVrsVariation: + if hgvs in errors_by_hgvs: + raise ValueError(errors_by_hgvs[hgvs]) + return FakeVrsVariation( + id_by_hgvs.get(hgvs, f"ga4gh:VA.{hgvs}"), + type_by_hgvs.get(hgvs, "Allele"), + ) + + return _translate + + +async def _map_variants(session, mock_worker_ctx, mapping_run, score_set, with_layers=frozenset({"g", "c", "p"})): + """Run the (mocked) mapping job so the score set's variants gain authoritative + MappingRecords/Alleles — the real inputs the reverse translation job reads.""" + + async def dummy_mapping_job(): + return await construct_mock_mapping_output(session=session, score_set=score_set, with_layers=with_layers) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], mapping_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + session.commit() + + +async def _reverse_translate(session, mock_worker_ctx, rt_run): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) + + +def _cross_level_statuses(session, score_set_id, status=None): + query = ( + session.query(VariantAnnotationStatus) + .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + .filter( + Variant.score_set_id == score_set_id, + VariantAnnotationStatus.annotation_type == "cross_level_translation", + ) + ) + if status is not None: + query = query.filter(VariantAnnotationStatus.status == status) + return query.all() + + +def _non_authoritative_links(session): + return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() + + +@pytest.mark.unit +class TestBuildTranslationConfig: + """Unit tests for _build_translation_config — the optional translation_config job param.""" + + def test_none_uses_job_defaults(self): + config = _build_translation_config(None) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_empty_dict_uses_job_defaults(self): + config = _build_translation_config({}) + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + def test_overrides_win_and_string_wt_codon_mode_is_coerced(self): + config = _build_translation_config({"wt_codon_mode": "unambiguous", "max_indel_size": 7}) + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + assert config.include_indels is True # untouched default preserved + + def test_can_disable_indels_with_none_mode(self): + config = _build_translation_config({"wt_codon_mode": "none", "include_indels": False}) + assert config.wt_codon_mode == WtCodonMode.NONE + assert config.include_indels is False + + def test_rejects_invalid_wt_codon_mode_with_actionable_message(self): + with pytest.raises(ValueError, match=r"wt_codon_mode 'bogus'.*Valid values"): + _build_translation_config({"wt_codon_mode": "bogus"}) + + def test_rejects_wt_codon_mode_without_indels(self): + # TranslationConfig enforces: a wt_codon_mode other than "none" requires include_indels. + with pytest.raises(ValueError, match="translation_config"): + _build_translation_config({"wt_codon_mode": "all", "include_indels": False}) + + def test_rejects_unknown_field_and_lists_valid_options(self): + with pytest.raises( + ValueError, match=r"Unknown translation_config option\(s\): not_a_real_field.*Valid options" + ): + _build_translation_config({"not_a_real_field": 1}) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestReverseTranslateVariantsForScoreSetUnit: + """Unit tests for the reverse_translate_variants_for_score_set job.""" + + async def test_no_mapping_records_is_a_noop( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no current, post-mapped mapping records there is nothing to translate.""" + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0} + + # No candidate alleles, links, or annotations were produced. + assert session.query(Allele).count() == 0 + assert _non_authoritative_links(session) == [] + assert _cross_level_statuses(session, sample_score_set.id) == [] + + async def test_creates_genomic_and_coding_candidate_alleles( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A mapped variant expands into one non-authoritative allele per equivalence-class + candidate, each linked to the originating mapping record.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} + + # Two non-authoritative links were created, both for our one mapping record. + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 2 + + genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() + assert genomic_allele.level == AnnotationLayer.genomic.value + assert genomic_allele.hgvs_g == g_candidate + assert genomic_allele.hgvs_c is None + assert genomic_allele.transcript == "NC_000001.11" + assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} + + coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() + assert coding_allele.level == AnnotationLayer.cdna.value + assert coding_allele.hgvs_c == c_candidate + assert coding_allele.hgvs_g is None + assert coding_allele.transcript == "NM_000001.1" + + # The candidate alleles are linked to the same mapping record as the authoritative allele. + mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() + assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + + statuses = _cross_level_statuses(session, sample_score_set.id) + assert len(statuses) == 1 + assert statuses[0].status == "success" + + async def test_transcript_is_queryable_via_derived_expression( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Allele.transcript is a derived hybrid_property — filtering on it in SQL resolves to + the accession of the populated HGVS column, with no stored transcript column.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + # The SQL expression derives the accession from the populated HGVS column. + genomic = session.query(Allele).filter(Allele.transcript == "NC_000001.11").all() + assert [a.vrs_digest for a in genomic] == ["ga4gh:VA.genomic"] + coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() + assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A bracketed genomic candidate (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" + construct = fake_construct({assay_hgvs: ([], [block_candidate])}) + translate = fake_translate( + {block_candidate: "ga4gh:CPB.block"}, + type_by_hgvs={block_candidate: "CisPhasedBlock"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() + assert block_allele.level == AnnotationLayer.genomic.value + # The full bracketed expression is stored verbatim; its accession anchors the row. + assert block_allele.hgvs_g == block_candidate + assert block_allele.transcript == "NC_000001.11" + assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} + + assert len(_non_authoritative_links(session)) == 1 + + async def test_duplicate_candidate_digests_are_deduped_per_record( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a coding and genomic candidate resolve to the same VRS digest, only one + allele/link is written for that mapping record (per the library's dedup contract).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + # Both candidates collapse to the same VRS digest. + translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert len(_non_authoritative_links(session)) == 1 + assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 + + async def test_candidate_equal_to_authoritative_allele_is_not_relinked( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A derived candidate whose digest equals the record's authoritative (assay-measured) + allele is not linked again — the record already links that allele authoritatively, and a + derived duplicate would surface the measured variant twice. No new link and no new allele; + an empty derived set is still a success with nothing to add.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + # Exactly one allele exists after mapping: the authoritative post-mapped allele. + assert session.query(Allele).count() == 1 + authoritative_allele = session.query(Allele).one() + assert authoritative_allele.vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [])}) + # The candidate resolves to the same digest as the authoritative allele. + translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 0} + + # The authoritative allele was reused, not duplicated. + assert session.query(Allele).count() == 1 + # No derived link was added: the candidate equals the record's authoritative allele. + assert _non_authoritative_links(session) == [] + + async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """An independent second run is idempotent: it retires the prior derived links (closing + valid_to) and writes fresh live ones, so the set of *current* non-authoritative links is + stable while the superseded links are retained as history — the partial unique index on + live links is never violated and no alleles are duplicated.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) + + # First run: two live derived links. + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + first = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert first.status == JobStatus.SUCCEEDED + first_links = _non_authoritative_links(session) + assert len(first_links) == 2 + assert all(link.valid_to is None for link in first_links) + + # Second, independent run with identical inputs. + rerun = JobRun( + urn="test:reverse_translate_variants_for_score_set:rerun", + job_type="reverse_translate_variants_for_score_set", + job_function="reverse_translate_variants_for_score_set", + max_retries=3, + retry_count=0, + job_params=dict(sample_independent_reverse_translation_run.job_params), + ) + session.add(rerun) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + second = await _reverse_translate(session, mock_worker_ctx, rerun) + + assert second.status == JobStatus.SUCCEEDED + + links = _non_authoritative_links(session) + live = [link for link in links if link.valid_to is None] + retired = [link for link in links if link.valid_to is not None] + + # The live set is stable (still exactly the two candidates) and the prior run's links are + # retained as closed history rather than deleted. + assert len(live) == 2 + assert len(retired) == 2 + + # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under + # one supersession timestamp — so a point-in-time query never lands in a hole between runs, + # even though the translation loop ran (and could have committed) between retire and insert. + assert {link.valid_to for link in retired} == {link.valid_from for link in live} + assert len({link.valid_from for link in live}) == 1 + + # The two candidate alleles are reused across runs, not duplicated. + assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 + + # The cross-level translation status is versioned the same way: prior retired, one current. + statuses = _cross_level_statuses(session, sample_score_set.id) + assert len(statuses) == 2 + assert len([s for s in statuses if s.current]) == 1 + + async def test_all_failures_fail_the_job( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """If every variant's translation errors, the job fails and records FAILED annotations.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + construct = fake_construct({}, errors_by_hgvs={assay_hgvs: "forward translation failed"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + + assert _non_authoritative_links(session) == [] + failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") + assert len(failed_statuses) == 1 + assert failed_statuses[0].error_message == "forward translation failed" + # Library/input-level failures still carry the assay-level HGVS as metadata. + assert failed_statuses[0].annotation_metadata == {"hgvs_input": assay_hgvs} + + async def test_partial_success_succeeds_with_mixed_annotations( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One variant translating and another erroring yields a SUCCEEDED job with both a + success and a failure annotation.""" + variant_ok = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + variant_err = Variant( + score_set_id=sample_score_set.id, + urn="variant:2", + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + ) + session.add_all([variant_ok, variant_err]) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct( + {"NM_000000.1:c.1A>G": ([], [g_candidate])}, + errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, + ) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} + + assert len(_cross_level_statuses(session, sample_score_set.id, status="success")) == 1 + assert len(_cross_level_statuses(session, sample_score_set.id, status="failed")) == 1 + assert len(_non_authoritative_links(session)) == 1 + + async def test_partial_candidate_translation_failure_keeps_success_with_metadata( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """One candidate translating and another failing VRS translation leaves the variant a + SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + good_candidate = "NC_000001.11:g.1000A>G" + bad_candidate = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: ([], [good_candidate, bad_candidate])}) + translate = fake_translate( + {good_candidate: "ga4gh:VA.genomic"}, + errors_by_hgvs={bad_candidate: "untranslatable form"}, + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert len(_non_authoritative_links(session)) == 1 + + statuses = _cross_level_statuses(session, sample_score_set.id, status="success") + assert len(statuses) == 1 + failed_candidates = statuses[0].annotation_metadata["failed_candidates"] + assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] + + async def test_all_candidates_failing_translation_marks_variant_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When every candidate fails VRS translation, the variant is FAILED, no allele is + created, and the per-candidate errors are retained in metadata.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + bad_candidate = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: ([], [bad_candidate])}) + translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.FAILED + assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} + assert _non_authoritative_links(session) == [] + + failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") + assert len(failed_statuses) == 1 + assert failed_statuses[0].error_message == "All candidate HGVS failed VRS translation." + metadata = failed_statuses[0].annotation_metadata + assert metadata["hgvs_input"] == assay_hgvs + assert metadata["failed_candidates"] == [ + {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} + ] + + async def test_no_coding_transcript_is_skipped_not_failed( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A target gene aligned only at the genomic level has no coding transcript, so its + variants are SKIPPED (no protein consequence) rather than attempted and failed.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript to anchor on. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + + # No translation attempted: no candidate alleles or links, and the variant is + # recorded as SKIPPED rather than FAILED. + assert _non_authoritative_links(session) == [] + assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] + assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + + async def test_translation_config_param_overrides_job_defaults( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A translation_config job param overrides the job's TranslationConfig defaults, and the + resulting config is passed through to construct_equivalent_variants.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + run = sample_independent_reverse_translation_run + run.job_params = {**run.job_params, "translation_config": {"wt_codon_mode": "unambiguous", "max_indel_size": 7}} + session.add(run) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.wt_codon_mode == WtCodonMode.UNAMBIGUOUS + assert config.max_indel_size == 7 + # Defaults the param did not override are preserved. + assert config.include_indels is True + + async def test_translation_config_defaults_when_param_absent( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """With no translation_config param the job falls back to its sensible defaults + (full codon equivalence class, indels included).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["config"] = config + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + config = captured["config"] + assert config.include_indels is True + assert config.wt_codon_mode == WtCodonMode.ALL + + async def test_protein_level_transcript_resolved_via_uta( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-only mapping has no cdna alignment transcript, so its coding transcript + is resolved NP_→NM_ via UTA and supplied as the VariantInput hint.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: no cdna TargetGeneMapping, so the transcript must come from UTA. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + + assay_hgvs = "NP_000000.1:p.Met1Val" + g_candidate = "NC_000001.11:g.1000A>G" + translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + + # Capture the transcript hint each VariantInput was built with, then delegate to the + # normal stub for the translation result. + captured_hints: dict[str, str | None] = {} + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000111.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + # The NP_ accession was resolved to its NM_ transcript and passed as the hint. + assert captured_hints == {assay_hgvs: "NM_000111.1"} From 7371d44b7b0cb84cb7ffc967fe06957c11d3b6e9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 8 Jun 2026 12:16:08 -0700 Subject: [PATCH 15/93] fix(reverse-translation): anchor cdna transcript lookup to mapping run date Previously the cdna-transcript lookup was keyed by target_gene_id alone, which meant a re-mapped score set could bind a stale NM_ transcript from an earlier run rather than the one the current run emitted. - Key cdna_transcript_by_run on (target_gene_id, mapped_date); within a key the highest-id row wins, so a same-run replacement takes precedence. - Carry mapped_date through the MappingRecord query so each record anchors to its own run's cdna row. - Add _TranscriptResolutionSkipReason to distinguish recoverable skips (protein-coding target, transcript unresolved) from correct skips (non-coding/regulatory target, no protein consequence). Emit skip_category in annotation_metadata. - Add target_gene_id to _TranscriptResolution so skip classification can look up the target's TargetCategory. - Add Mapped[] type annotations to TargetGene.id and .category columns. - New tests: genomic-accession coding target RT, latest-cdna-row-within- run selection, stale-cdna-row isolation, skip classification (coding recoverable vs. regulatory correct), and cdna TGM persistence in the mapping job. Refs mavedb-api#763 --- src/mavedb/models/target_gene.py | 4 +- .../variant_processing/reverse_translation.py | 117 ++++++-- .../jobs/variant_processing/test_mapping.py | 45 ++++ .../test_reverse_translation.py | 250 ++++++++++++++++++ 4 files changed, 397 insertions(+), 19 deletions(-) diff --git a/src/mavedb/models/target_gene.py b/src/mavedb/models/target_gene.py index 671a4380f..5419c3478 100644 --- a/src/mavedb/models/target_gene.py +++ b/src/mavedb/models/target_gene.py @@ -23,10 +23,10 @@ class TargetGene(Base): __tablename__ = "target_genes" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) name = Column(String, nullable=False) - category = Column( + category: Mapped[TargetCategory] = Column( Enum(TargetCategory, create_constraint=True, length=32, native_enum=False, validate_strings=True), nullable=False, ) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 61ca1d883..c5f14bf68 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -12,6 +12,8 @@ import functools import logging import os +from datetime import date +from enum import Enum from typing import Any, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator @@ -30,6 +32,7 @@ from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.target_category import TargetCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -62,6 +65,40 @@ class _TranscriptResolution(NamedTuple): variant: Variant gene_transcript: str | None protein_accession: str | None + target_gene_id: int | None + + +class _TranscriptResolutionSkipReason(Enum): + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" + NO_CODING_TRANSCRIPT = "no_coding_transcript" + + @classmethod + def classify( + cls, resolution: _TranscriptResolution, category: TargetCategory | None + ) -> tuple["_TranscriptResolutionSkipReason", str]: + """Classify why a record was skipped, distinguishing recoverable from correct skips. + + A protein-coding target with no resolvable transcript is *recoverable* (it should now be + rare -- the mapper selects a MANE transcript for coding genomic-accession targets); a + regulatory / non-coding target has no protein consequence and is a *correct* skip. + """ + if not resolution.rec.hgvs_assay_level: + return ( + cls.NO_ASSAY_LEVEL_HGVS, + "No assay-level HGVS available to reverse-translate.", + ) + if category == TargetCategory.protein_coding: + return ( + cls.TRANSCRIPT_UNRESOLVED, + "Protein-coding target but no coding transcript could be resolved " + "(no cdna TargetGeneMapping and no NP_->NM_ association). Recoverable: " + "re-map or check transcript selection.", + ) + return ( + cls.NO_CODING_TRANSCRIPT, + "Non-coding/regulatory target has no protein consequence to reverse-translate.", + ) def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: @@ -169,25 +206,55 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - # Build {target_gene_id -> NM_ transcript} from the cdna TargetGeneMappings the mapper - # emits. Reverse translation must run against the cdna (NM_) transcript. - cdna_transcript_by_gene: dict[int, str | None] = dict( - job_manager.db.execute( - select(TargetGeneMapping.target_gene_id, TargetGeneMapping.reference_accession) + # Build {(target_gene_id, run date) -> NM_ transcript} from the cdna TargetGeneMappings + # the mapper emits. Reverse translation must run against the cdna (NM_) transcript. + # + # TargetGeneMapping is not valid-time-versioned and re-mapping cannot delete prior rows + # (retired MappingRecords still FK them), so a re-mapped target accumulates several cdna + # rows. There is no run anchor on the row, so we approximate one with mapped_date and key + # the lookup by (target gene, run date): each record resolves only the cdna transcript + # from *its own* run, matched below against the run date of the genomic TargetGeneMapping + # the record points to. All of a run's TargetGeneMappings share one mapped_date (stamped + # once per job), so a record and its run's cdna row match even if the job spans midnight. + # Within a single key, order by id so the newest row wins. + # + # TODO#763: mapped_date is day-granular, so two re-maps on the *same calendar day* collide + # on the key. If a later same-day re-map emits no cdna row (e.g. transcript selection hit + # a transient Ensembl/gene-normalizer failure, or the target genuinely lost its coding + # transcript) but an earlier same-day one did, RT can still bind the earlier (stale) + # transcript. This keying narrows the window from "any prior run ever" to "a same-day + # re-map" -- the bug now requires re-runs in quick succession -- but only an actual run + # anchor / versioned home for reference identity closes it fully. + cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { + (target_gene_id, mapped_date): reference_accession + for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( + select( + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + TargetGeneMapping.reference_accession, + ) .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) .where(TargetGene.score_set_id == score_set_id) .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) .where(TargetGeneMapping.reference_accession.isnot(None)) + .order_by(TargetGeneMapping.id) ) .tuples() .all() - ) + } # Load current, authoritative, and successfully-mapped MappingRecords along with their # target gene (for the coding-transcript lookup) and parent Variant. - rows: Sequence[tuple[MappingRecord, Variant, int]] = ( + # The joined TargetGeneMapping is the record's own (genomic) mapping row, so its + # mapped_date is this record's run date -- used to anchor the cdna lookup to the run. + rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( job_manager.db.execute( - select(MappingRecord, Variant, TargetGeneMapping.target_gene_id) + select( + MappingRecord, + Variant, + TargetGeneMapping.target_gene_id, + TargetGeneMapping.mapped_date, + ) .join(MappingRecordAllele, MappingRecord.id == MappingRecordAllele.mapping_record_id) .join(Variant, MappingRecord.variant_id == Variant.id) .outerjoin(TargetGeneMapping, MappingRecord.target_gene_mapping_id == TargetGeneMapping.id) @@ -214,8 +281,8 @@ async def reverse_translate_variants_for_score_set( # resolve NP_→NM_ from UTA in a single batch query below. transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() - for rec, variant, target_gene_id in rows: - coding_accession = cdna_transcript_by_gene.get(target_gene_id) + for rec, variant, target_gene_id, mapped_date in rows: + coding_accession = cdna_transcript_by_run.get((target_gene_id, mapped_date)) protein_accession = None if not coding_accession and rec.hgvs_assay_level is not None: raw_accession = extract_accession(rec.hgvs_assay_level) @@ -223,10 +290,23 @@ async def reverse_translate_variants_for_score_set( protein_accession = raw_accession protein_accessions.add(raw_accession) - transcript_resolutions.append(_TranscriptResolution(rec, variant, coding_accession, protein_accession)) + transcript_resolutions.append( + _TranscriptResolution(rec, variant, coding_accession, protein_accession, target_gene_id) + ) transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) + # Target gene category per gene, so a skip can be classified honestly: a coding target + # with no resolvable transcript is recoverable (should now be rare), while a regulatory / + # non-coding target has no protein consequence and is a correct skip. + target_category_by_gene: dict[int, TargetCategory] = dict( + job_manager.db.execute( + select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) + ) + .tuples() + .all() + ) + # Build VariantInputs, supplying the resolved coding transcript for every input # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence # anchored on that transcript). Object identity on each VariantInput is preserved @@ -237,13 +317,13 @@ async def reverse_translate_variants_for_score_set( # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. variant_inputs: list[Any] = [] variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} - skipped_variants: list[tuple[MappingRecord, Variant]] = [] + skipped_variants: list[_TranscriptResolution] = [] for p in transcript_resolutions: transcript = p.gene_transcript or ( transcript_by_protein.get(p.protein_accession) if p.protein_accession else None ) if not transcript or not p.rec.hgvs_assay_level: - skipped_variants.append((p.rec, p.variant)) + skipped_variants.append(p) continue inp = VariantInput(hgvs=p.rec.hgvs_assay_level, transcript=transcript) @@ -429,15 +509,18 @@ async def reverse_translate_variants_for_score_set( ) skipped = len(skipped_variants) - for rec, variant in skipped_variants: + for p in skipped_variants: + category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None + skip_category, reason = _TranscriptResolutionSkipReason.classify(p, category) annotation_manager.add_annotation( - variant_id=variant.id, + variant_id=p.variant.id, annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, status=AnnotationStatus.SKIPPED, annotation_data={ "annotation_metadata": { - "hgvs_input": rec.hgvs_assay_level, - "reason": "No coding transcript for target gene; no protein consequence to reverse-translate.", + "hgvs_input": p.rec.hgvs_assay_level, + "skip_category": skip_category.value, + "reason": reason, } }, ) diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index b28f61564..74c9acb1c 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -18,6 +18,7 @@ from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -479,6 +480,50 @@ async def dummy_mapping_job(): assert annotation_statuses[0].annotation_type == "vrs_mapping" assert annotation_statuses[0].status == "success" + async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """An identity cdna TargetGeneMapping (a cdna layer with no per-variant scores + joining it) persists with its reference_accession (NM_) and null QC/counts -- the + artifact reverse translation consumes to resolve the projection transcript.""" + variant = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + + # Genomic assay layer + an identity cdna layer (no cdna per-variant scores). + async def dummy_mapping_job(): + return await construct_mock_mapping_output( + session=session, score_set=sample_score_set, with_layers={"g", "c"} + ) + + with patch.object(_UnixSelectorEventLoop, "run_in_executor", return_value=dummy_mapping_job()): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + assert result.status == JobStatus.SUCCEEDED + + cdna_tgms = ( + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == AnnotationLayer.cdna).all() + ) + assert len(cdna_tgms) == len(sample_score_set.target_genes) + tgm = cdna_tgms[0] + assert tgm.reference_accession == "NM_999999.1" + # Identity row: no mapped_variant joins it, so QC/counts are null. + assert tgm.total_variants is None + assert tgm.variants_mapped_cleanly is None + assert tgm.percent_identity is None + async def test_map_variants_for_score_set_success_no_successful_mapping( self, session, diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index c7e434845..459ec53db 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -5,6 +5,7 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from datetime import timedelta from unittest.mock import patch from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode @@ -13,9 +14,11 @@ from mavedb.models.allele import Allele from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.job_pipeline import JobStatus +from mavedb.models.enums.target_category import TargetCategory from mavedb.models.job_run import JobRun from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set @@ -771,6 +774,253 @@ async def test_no_coding_transcript_is_skipped_not_failed( assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + async def test_genomic_accession_coding_target_is_reverse_translated( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A genomic-accession (NC_:g.) coding variant -- previously skipped for want of a + coding transcript -- is reverse-translated once the mapper emits a cdna + TargetGeneMapping, whose reference_accession anchors the projection.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity TargetGeneMapping (carrying the NM_); the + # genomic variant projects onto that transcript for reverse translation. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + + assay_hgvs = "NC_000001.11:g.1000A>G" + g_candidate = "NC_000001.11:g.1000A>G" + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: ([], [g_candidate])})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.genomic"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert _cross_level_statuses(session, sample_score_set.id, status="skipped") == [] + + def _run_mapped_date(self, session, target_gene_id): + """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" + return ( + session.query(TargetGeneMapping) + .filter( + TargetGeneMapping.target_gene_id == target_gene_id, + TargetGeneMapping.alignment_level == AnnotationLayer.genomic, + ) + .first() + .mapped_date + ) + + async def test_uses_latest_cdna_row_within_the_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """When a run has more than one cdna TargetGeneMapping for a target (same run date), + RT reverse-translates against the newest (highest id), not an arbitrary one.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Mapping emits a cdna TargetGeneMapping (NM_999999.1) stamped with the run date. + await _map_variants( + session, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + with_layers={"g", "c"}, + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A newer cdna row (higher id) for the same target and same run date. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=AnnotationLayer.cdna, + reference_accession="NM_111111.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date, + ) + ) + session.commit() + + assay_hgvs = "NC_000001.11:g.1000A>G" + g_candidate = "NC_000001.11:g.1000A>G" + delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + captured: dict = {} + + def capturing_construct(inputs, *, transcripts, coordinates, config): + captured["transcripts"] = [inp.transcript for inp in inputs] + return delegate(inputs, transcripts=transcripts, coordinates=coordinates, config=config) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", capturing_construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.g"})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + # The newest cdna row within the run wins, not the first-mapped one. + assert captured["transcripts"] == ["NM_111111.1"] + + async def test_ignores_stale_cdna_row_from_a_different_run( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A cdna row left behind by a *different* run (different run date) is not used: + the current run emitted no cdna row, so the variant is skipped (transcript + unresolved) rather than reverse-translated against the stale transcript -- the + mapped_date anchor narrows the accumulation edge case (mavedb-api#763).""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Current run maps genomic only -- no cdna row for this run's date. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + target_gene = sample_score_set.target_genes[0] + run_date = self._run_mapped_date(session, target_gene.id) + # A leftover cdna row from a *prior* run (an earlier date) must not be picked up. + session.add( + TargetGeneMapping( + target_gene_id=target_gene.id, + alignment_level=AnnotationLayer.cdna, + reference_accession="NM_888888.1", + preferred=False, + tool_version="pytest.0.0", + mapped_date=run_date - timedelta(days=1), + ) + ) + session.commit() + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + + async def test_coding_target_skip_is_classified_recoverable( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A protein-coding target with no resolvable coding transcript is a *recoverable* + skip -- classified ``transcript_unresolved`` so it is findable, distinct from a + regulatory target's correct skip.""" + assert sample_score_set.target_genes[0].category == TargetCategory.protein_coding + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic-only mapping: no cdna TargetGeneMapping, so no coding transcript resolves. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + + async def test_regulatory_target_skip_is_classified_correct( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A non-coding/regulatory target has no protein consequence: its skip is *correct*, + classified ``no_coding_transcript`` rather than the recoverable category.""" + target = sample_score_set.target_genes[0] + target.category = TargetCategory.regulatory + session.add(target) + session.commit() + + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g"} + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({})), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({})), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} + skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + assert len(skipped) == 1 + assert skipped[0].annotation_metadata["skip_category"] == "no_coding_transcript" + async def test_translation_config_param_overrides_job_defaults( self, session, From 181360f1f35a813f6ea08d26668a05b9354cbe06 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 11:53:58 -0700 Subject: [PATCH 16/93] feat(hgvs): add strip_protein_prediction_parens utility Forward translation emits predicted protein consequences in parentheses (e.g. p.(Ala222Val)). The parens denote inference, not a distinct variant form, so normalize to the bare p. form before storing. Strings without prediction parens are returned unchanged. Includes parametrized unit tests. --- src/mavedb/lib/hgvs.py | 14 ++++++++++++++ tests/lib/test_hgvs.py | 22 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index f330bdfd4..5826e67c3 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -22,6 +22,20 @@ def extract_accession(hgvs_string: str) -> str: return token.split(":", 1)[0].strip() +_HGVS_P_PREDICTION = re.compile(r":p\.\((.+)\)\s*$") + + +def strip_protein_prediction_parens(hgvs_p: str) -> str: + """Unwrap the prediction parentheses from a protein HGVS: ``p.(Ala222Val)`` -> ``p.Ala222Val``. + + Forward translation (``c_to_p``) emits a *predicted* consequence in parentheses. ga4gh + translates either form fine, but the parens are noise we don't want in the stored HGVS -- + they denote inference, not a different variant -- so we normalize to the bare form for + consistency. A string with no prediction parens is returned unchanged. + """ + return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) + + def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index 355174fb1..45793034a 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -1,6 +1,26 @@ import pytest -from mavedb.lib.hgvs import extract_accession, join_cis_phased_hgvs, split_cis_phased_hgvs +from mavedb.lib.hgvs import ( + extract_accession, + join_cis_phased_hgvs, + split_cis_phased_hgvs, + strip_protein_prediction_parens, +) + + +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + ("NP_000456.2:p.(Ala222Val)", "NP_000456.2:p.Ala222Val"), + ("NP_000456.2:p.(Tyr745Ter)", "NP_000456.2:p.Tyr745Ter"), + ("NP_000456.2:p.(Ala222_Val225del)", "NP_000456.2:p.Ala222_Val225del"), + # no prediction parens -> unchanged + ("NP_000456.2:p.Ala222Val", "NP_000456.2:p.Ala222Val"), + ("NM_000001.1:c.5A>G", "NM_000001.1:c.5A>G"), + ], +) +def test_strip_protein_prediction_parens(hgvs, expected): + assert strip_protein_prediction_parens(hgvs) == expected @pytest.mark.parametrize( From bd6f6a5e0e7a24f0cb4279edbe4a0e3fbe3d639b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 11:54:59 -0700 Subject: [PATCH 17/93] fix(vrs-utils): normalize and re-identify VRS alleles to prevent digest collisions - Re-identify each component via normalize_and_identify after translate_from so the stored vrs_digest reflects current content, not a stale value cached by the reused AlleleTranslator's Merkle tree. Without this, distinct biological variants at the same position (e.g. A>C and A>T) share one digest and are merged by the digest-keyed get_or_create_allele. - Use ga4gh_identify with in_place="always" in identify_allele and identify_variation so an allele that already carries a translator-stamped id has it recomputed, not retained. - Coerce ReferenceLengthExpression -> LiteralSequenceExpression in normalize_and_identify, mirroring dcd_mapping's _rle_to_lse, so RT-built alleles hash identically to the mapper's authoritative alleles and dedup correctly across sources. - Add regression tests: stale-id overwrite, distinct-alt digest isolation, cis-phased ordering canonicalization, and RLE->LSE coercion. --- src/mavedb/lib/vrs_utils.py | 61 ++++++++++++++++-- tests/lib/test_vrs_utils.py | 120 ++++++++++++++++++++++++++++++++++-- 2 files changed, 170 insertions(+), 11 deletions(-) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 7872f2ed6..052c4b702 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -9,11 +9,18 @@ current content. """ +from itertools import cycle from typing import Any from ga4gh.core import ga4gh_identify from ga4gh.vrs.extras.translator import AlleleTranslator -from ga4gh.vrs.models import Allele, CisPhasedBlock, LiteralSequenceExpression, SequenceLocation +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + LiteralSequenceExpression, + ReferenceLengthExpression, + SequenceLocation, +) from ga4gh.vrs.normalize import normalize from mavedb.lib.hgvs import split_cis_phased_hgvs @@ -62,11 +69,22 @@ def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Alle identifies to one ``ga4gh:CPB.`` digest and dedups to a single row regardless of component ordering. + Every component is normalized and re-identified through :func:`normalize_and_identify` + before use. ``translate_from`` is called with ``do_normalize=False`` and stamps ``id`` via + plain ``ga4gh_identify`` on the reused translator, so without this step a component carries a + non-canonical digest computed from the unnormalized object and from the Merkle tree's cached + sub-object digests — distinct biological variants can then collide onto one ``vrs_digest`` and + be merged by the digest-keyed ``get_or_create_allele``. Normalizing here also keeps RT digests + consistent with the mapper's, which is what lets the same allele dedup across sources. + :param hgvs: a single- or cis-phased-multivariant HGVS string :param translator: caller-owned AlleleTranslator reused across calls :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set """ - members = [translate_hgvs_to_vrs(component, translator) for component in split_cis_phased_hgvs(hgvs)] + members = [ + normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + for component in split_cis_phased_hgvs(hgvs) + ] if len(members) == 1: return members[0] @@ -83,12 +101,18 @@ def identify_allele(allele: Allele) -> str: the location digest and the allele digest first ensures the id is always derived from the current object content — not from a value set before a refgetAccession mutation or normalization. + + ``id`` is recomputed as well: ``ga4gh_identify`` returns a non-empty ``id`` as-is + (its default ``in_place`` only fills an *empty* id), so an allele that already + carries one — e.g. stamped by an ``identify=True`` ``AlleleTranslator`` — would + otherwise keep its stale id even with the digests cleared. Use in_place="always" + to force it to be recomputed from the content-derived digest. """ if isinstance(allele.location, SequenceLocation): allele.location.digest = None allele.digest = None - digest = ga4gh_identify(allele) + digest = ga4gh_identify(allele, in_place="always") if digest is None: raise ValueError("Failed to compute GA4GH identifier for allele") # noqa: EM101 @@ -113,7 +137,7 @@ def identify_variation(variation: Allele | CisPhasedBlock) -> str: member.digest = None variation.digest = None - digest = ga4gh_identify(variation) + digest = ga4gh_identify(variation, in_place="always") if digest is None: raise ValueError("Failed to compute GA4GH identifier for variation") # noqa: EM101 @@ -123,13 +147,40 @@ def identify_variation(variation: Allele | CisPhasedBlock) -> str: def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """Normalize *allele* and stamp it with a freshly computed GA4GH digest. - Pairs the two finalize steps every VRS allele construction path needs. + Pairs the finalize steps every VRS allele construction path needs. Routing identification through :func:`identify_allele` (rather than ``ga4gh_identify`` directly) is the invariant that protects against the Merkle-tree's stale-digest behavior after mutation -- so any allele construction site that bypasses this helper risks reintroducing the stale-digest bug. + + Normalization can leave an indel as a ``ReferenceLengthExpression``; this coerces + it to a ``LiteralSequenceExpression`` so the result matches dcd_mapping's + authoritative alleles (``vrs_map._rle_to_lse``), which always store LSE. Without + the coercion the same biological indel hashes to two digests -- RLE here, LSE from + the mapper -- so reverse translation's regenerated genomic form fails to dedup + against the authoritative row and a duplicate allele is linked. """ allele = normalize(allele, data_proxy=data_proxy) + if isinstance(allele.state, ReferenceLengthExpression): + allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) allele.id = identify_allele(allele) return allele + + +def _rle_to_lse( + rle: ReferenceLengthExpression, location: SequenceLocation, data_proxy: Any +) -> LiteralSequenceExpression: + """Coerce a ReferenceLengthExpression to an equivalent LiteralSequenceExpression. + + Mirrors ``dcd_mapping:vrs_map.py::_rle_to_lse`` byte-for-byte so an allele built here + hashes identically to the mapper's authoritative allele for the same variant. Derives + the literal sequence by tiling the repeat subunit out to ``rle.length``. + """ + sequence_id = location.sequenceReference.refgetAccession + start: int = location.start + end = start + rle.repeatSubunitLength + subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) + c = cycle(subsequence) + derived_sequence = "".join(next(c) for _ in range(rle.length)) + return LiteralSequenceExpression(sequence=derived_sequence) diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index 6f2bd382f..f40dce976 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -1,5 +1,7 @@ # ruff: noqa: E402 +from types import SimpleNamespace + import pytest pytest.importorskip("ga4gh.vrs") @@ -8,15 +10,25 @@ Allele, CisPhasedBlock, LiteralSequenceExpression, + ReferenceLengthExpression, SequenceLocation, SequenceReference, ) from mavedb.lib import vrs_utils -from mavedb.lib.vrs_utils import identify_variation, translate_hgvs_to_variation +from mavedb.lib.vrs_utils import ( + _rle_to_lse, + identify_variation, + normalize_and_identify, + translate_hgvs_to_variation, +) _SQ = "SQ." + "a" * 32 +# A digest a reused ga4gh AlleleTranslator might leave on a component before it is +# re-identified — stale from the Merkle cache, or carried over from a sibling variant. +_STALE_ID = "ga4gh:VA." + "Z" * 32 + def _allele(start: int, alt: str) -> Allele: return Allele( @@ -38,14 +50,72 @@ def _fake(hgvs: str, translator=None) -> Allele: monkeypatch.setattr(vrs_utils, "translate_hgvs_to_vrs", _fake) +def _stub_normalize(monkeypatch) -> None: + """Make normalization a no-op so identification can be exercised without a seqrepo. + + Identification (the regression surface) runs for real; only the proxy-backed + normalize step is stubbed out. + """ + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: allele) + + +def _translator() -> SimpleNamespace: + return SimpleNamespace(data_proxy=object()) + + def test_single_variant_returns_a_bare_allele(monkeypatch): allele = _allele(1000, "G") _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) - result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=None) + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) assert isinstance(result, Allele) - assert result is allele + assert not isinstance(result, CisPhasedBlock) + + +def test_single_variant_is_reidentified_not_left_with_translator_digest(monkeypatch): + # ga4gh's translate_from (do_normalize=False) stamps id via plain ga4gh_identify on + # the reused translator; translate_hgvs_to_variation must re-identify so the persisted + # vrs_digest reflects current content rather than that carried-over value. + allele = _allele(1000, "G") + allele.id = _STALE_ID + _patch_component_translator(monkeypatch, {"NC_000001.11:g.1000A>G": allele}) + _stub_normalize(monkeypatch) + + result = translate_hgvs_to_variation("NC_000001.11:g.1000A>G", translator=_translator()) + + assert result.id is not None + assert result.id != _STALE_ID + assert result.id.startswith("ga4gh:VA.") + + +def test_same_position_distinct_alts_get_distinct_digests(monkeypatch): + # Regression: NC_000002.12:g.214809459A>C and g.214809459A>T are different MaveDB + # variants that cannot encode the same codon, yet were merged onto one Allele row + # (and thus one coding equivalent like NM_000465.4:c.109_111delinsCGA). The single + # component returned by the reused translator carried a stale/shared digest, which + # the digest-keyed get_or_create_allele then deduplicated. Re-identification must + # give distinct content distinct digests so they never collide. + a_c = _allele(214809458, "C") + a_t = _allele(214809458, "T") + a_c.id = a_t.id = _STALE_ID # what the unfixed path would persist for both + _patch_component_translator( + monkeypatch, + { + "NC_000002.12:g.214809459A>C": a_c, + "NC_000002.12:g.214809459A>T": a_t, + }, + ) + _stub_normalize(monkeypatch) + translator = _translator() + + res_c = translate_hgvs_to_variation("NC_000002.12:g.214809459A>C", translator=translator) + res_t = translate_hgvs_to_variation("NC_000002.12:g.214809459A>T", translator=translator) + + assert res_c.id != _STALE_ID + assert res_t.id != _STALE_ID + assert res_c.id != res_t.id def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): @@ -54,8 +124,9 @@ def test_cis_phased_multivariant_returns_an_identified_block(monkeypatch): "NC_000001.11:g.1002T>C": _allele(1002, "C"), } _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) - result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) + result = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) assert isinstance(result, CisPhasedBlock) assert len(result.members) == 2 @@ -68,9 +139,10 @@ def test_cis_phased_block_digest_is_order_independent(monkeypatch): "NC_000001.11:g.1002T>C": _allele(1002, "C"), } _patch_component_translator(monkeypatch, members) + _stub_normalize(monkeypatch) - forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=None) - reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=None) + forward = translate_hgvs_to_variation("NC_000001.11:g.[1000A>G;1002T>C]", translator=_translator()) + reverse = translate_hgvs_to_variation("NC_000001.11:g.[1002T>C;1000A>G]", translator=_translator()) # The same biological cis-phased set dedups to one row regardless of component ordering. assert forward.id == reverse.id @@ -84,3 +156,39 @@ def test_identify_variation_clears_stale_block_digest(): assert digest != "STALE" assert digest.startswith("ga4gh:CPB.") # recomputed from content, not the stale cache + + +def _rle_allele(start: int, *, length: int, repeat_subunit_length: int) -> Allele: + return Allele( + location=SequenceLocation( + sequenceReference=SequenceReference(refgetAccession=_SQ), + start=start, + end=start + repeat_subunit_length, + ), + state=ReferenceLengthExpression(length=length, repeatSubunitLength=repeat_subunit_length), + ) + + +def test_rle_to_lse_tiles_repeat_subunit(): + # repeatSubunitLength=2 reads 2 bases from the proxy; length=4 tiles them out to "ACAC". + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = _rle_to_lse(rle, location, data_proxy) + + assert isinstance(result, LiteralSequenceExpression) + assert result.sequence.root == "ACAC" + + +def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): + # Regression: normalization can yield an RLE for an indel, but dcd_mapping stores LSE; the + # finalize step must coerce so the same variant doesn't hash to two digests (and duplicate). + rle = _rle_allele(10, length=2, repeat_subunit_length=2) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: rle) + data_proxy = SimpleNamespace(get_sequence=lambda identifier, start, end: "AC") + + result = normalize_and_identify(rle, data_proxy=data_proxy) + + assert isinstance(result.state, LiteralSequenceExpression) + assert result.id is not None and result.id.startswith("ga4gh:VA.") From c17caf71499d0f522afcb314cb0805529cfbd23b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 13:23:36 -0700 Subject: [PATCH 18/93] refactor(reverse-translation): add p. alleles to equivalence set and tighten comments - Emit the protein consequence (result.hgvs_p) as a protein-level member of the equivalence set alongside the coding/genomic candidates. Prediction parens (p.(Ala222Val)) are stripped via strip_protein_prediction_parens before translation and storage. Protein-assay inputs are excluded (the protein is already the authoritative allele). - Trim all inline and docstring comments to essential why; remove redundant prose throughout reverse_translation.py. - Add tests: protein allele persistence, skip-classification for all three skip categories (no_assay_level_hgvs, transcript_unresolved, no_coding_transcript). --- .../variant_processing/reverse_translation.py | 147 +++++++----------- .../test_reverse_translation.py | 56 ++++++- 2 files changed, 109 insertions(+), 94 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index c5f14bf68..872044e91 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -2,7 +2,8 @@ For each mapped variant in a score set, calls construct_equivalent_variants from the variant-annotation library to produce all coding and genomic HGVS candidates -encoding the same protein consequence. The candidates are written as non-authoritative +encoding the same protein consequence, plus that protein consequence itself. The +candidates (coding, genomic, and the protein) are written as non-authoritative Allele rows linked to the existing MappingRecord via MappingRecordAllele. """ @@ -24,7 +25,7 @@ from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.hgvs import extract_accession +from mavedb.lib.hgvs import extract_accession, strip_protein_prediction_parens from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation @@ -46,9 +47,8 @@ logger = logging.getLogger(__name__) -# Job defaults when no `translation_config` override is supplied: enumerate the full coding -# equivalence class — every synonymous codon, indels included. wt_codon_mode "all" requires -# include_indels=True (TranslationConfig enforces this). +# Job defaults: full coding equivalence class — every synonymous codon, indels included. +# wt_codon_mode "all" requires include_indels=True. _DEFAULT_TRANSLATION_CONFIG: dict[str, Any] = { "include_indels": True, "wt_codon_mode": WtCodonMode.ALL, @@ -56,10 +56,9 @@ class _TranscriptResolution(NamedTuple): - """A mapping record paired with what we know about its coding transcript before the - batched UTA lookup: the gene-level cdna transcript if the mapper supplied one, and the - protein accession still awaiting NP_→NM_ resolution otherwise. Exactly one of the two is - set for a resolvable record; both are None when neither path applies (record is skipped).""" + """Pairs a mapping record with its pre-UTA transcript info: gene_transcript if the mapper + supplied a cdna row, protein_accession if NP_→NM_ resolution is still needed. Both None + means the record will be skipped.""" rec: MappingRecord variant: Variant @@ -77,11 +76,10 @@ class _TranscriptResolutionSkipReason(Enum): def classify( cls, resolution: _TranscriptResolution, category: TargetCategory | None ) -> tuple["_TranscriptResolutionSkipReason", str]: - """Classify why a record was skipped, distinguishing recoverable from correct skips. + """Classify why a record was skipped. - A protein-coding target with no resolvable transcript is *recoverable* (it should now be - rare -- the mapper selects a MANE transcript for coding genomic-accession targets); a - regulatory / non-coding target has no protein consequence and is a *correct* skip. + Protein-coding target with no transcript → recoverable (transcript_unresolved). + Non-coding/regulatory target → correct skip (no_coding_transcript). """ if not resolution.rec.hgvs_assay_level: return ( @@ -102,14 +100,11 @@ def classify( def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: - """Resolve each RefSeq protein accession (NP_/XP_…) to its preferred coding transcript - via UTA's associated_accessions table. - - Protein-level mappings carry no coding transcript — the mapper emits reference_accession - only for cdna alignments — so reverse-translating them relies on the NP_→NM_ association - UTA records. Resolution and the multi-transcript preference order both live in the - variant-annotation library's UtaClient; we own only the connection lifecycle here so the - long-lived worker process does not leak UTA connections across jobs. + """Resolve RefSeq protein accessions (NP_/XP_) to their preferred coding transcripts via UTA. + + Protein-level mappings carry no cdna transcript in TargetGeneMapping, so reverse + translation falls back to the NP_→NM_ association in UTA. Connection is opened and + closed per-call to avoid leaking connections across long-lived worker jobs. """ if not protein_accessions: return {} @@ -128,17 +123,11 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationConfig: - """Build the variant-annotation TranslationConfig from optional job-param overrides. - - Each key in `overrides` is a TranslationConfig field (that dataclass is the source of truth - for the available knobs) and wins over the job defaults in _DEFAULT_TRANSLATION_CONFIG; - absent or None means use the defaults. wt_codon_mode is coerced to the enum so a JSON string - value ("all"/"unambiguous"/"none") works. + """Build a TranslationConfig from optional job-param overrides merged with job defaults. - Raises ValueError with an actionable message — listing the offending value and the valid - options — for an unknown field, an invalid wt_codon_mode, or an invalid combination (e.g. a - wt_codon_mode other than "none" without include_indels). The allowed-field set is derived - from TranslationConfig itself, so it never drifts from the library. + Overrides win over _DEFAULT_TRANSLATION_CONFIG; wt_codon_mode accepts string values and + is coerced to the enum. Raises ValueError for unknown fields, invalid modes, or invalid + combinations (e.g. wt_codon_mode != "none" without include_indels). """ config_kwargs: dict[str, Any] = {**_DEFAULT_TRANSLATION_CONFIG, **(overrides or {})} @@ -172,20 +161,12 @@ async def reverse_translate_variants_for_score_set( ) -> JobExecutionOutcome: """Build the cross-level HGVS equivalence class for every mapped variant in the score set. - Reads current MappingRecords that carry an hgvs_assay_level string, collapses each - to its ProteinConsequence, and expands to all coding/genomic HGVS candidates via a - single batched subprocess call to the variant-annotation library. Each candidate is - written as a non-authoritative Allele linked to the MappingRecord. - - Required job_params: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking + For each current MappingRecord with an hgvs_assay_level, collapses to a ProteinConsequence + and expands to all coding/genomic HGVS candidates via the variant-annotation library. + Each candidate is written as a non-authoritative Allele linked to the MappingRecord. - Optional job_params: - - translation_config (dict): Overrides for any variant-annotation TranslationConfig - field (e.g. include_indels, wt_codon_mode, max_indel_size). Omitted keys fall back to - the job defaults in _DEFAULT_TRANSLATION_CONFIG (full codon equivalence class, indels - included). + job_params: score_set_id (int), correlation_id (str), + translation_config (dict, optional) — TranslationConfig overrides. """ job = job_manager.get_job() validate_job_params(["score_set_id", "correlation_id"], job) @@ -206,25 +187,14 @@ async def reverse_translate_variants_for_score_set( job_manager.update_progress(0, 100, "Starting reverse translation job.") logger.info(msg="Started reverse translation job.", extra=job_manager.logging_context()) - # Build {(target_gene_id, run date) -> NM_ transcript} from the cdna TargetGeneMappings - # the mapper emits. Reverse translation must run against the cdna (NM_) transcript. + # Build {(target_gene_id, run date) -> NM_ transcript} from cdna TargetGeneMappings. + # TargetGeneMapping rows accumulate across re-maps (retired records still FK them), so we + # key by (target_gene_id, mapped_date) to bind each record to its own run's cdna row. + # All TargetGeneMappings in one job share a mapped_date; within a key, highest id wins. # - # TargetGeneMapping is not valid-time-versioned and re-mapping cannot delete prior rows - # (retired MappingRecords still FK them), so a re-mapped target accumulates several cdna - # rows. There is no run anchor on the row, so we approximate one with mapped_date and key - # the lookup by (target gene, run date): each record resolves only the cdna transcript - # from *its own* run, matched below against the run date of the genomic TargetGeneMapping - # the record points to. All of a run's TargetGeneMappings share one mapped_date (stamped - # once per job), so a record and its run's cdna row match even if the job spans midnight. - # Within a single key, order by id so the newest row wins. - # - # TODO#763: mapped_date is day-granular, so two re-maps on the *same calendar day* collide - # on the key. If a later same-day re-map emits no cdna row (e.g. transcript selection hit - # a transient Ensembl/gene-normalizer failure, or the target genuinely lost its coding - # transcript) but an earlier same-day one did, RT can still bind the earlier (stale) - # transcript. This keying narrows the window from "any prior run ever" to "a same-day - # re-map" -- the bug now requires re-runs in quick succession -- but only an actual run - # anchor / versioned home for reference identity closes it fully. + # TODO#763: mapped_date is day-granular, so two re-maps on the same calendar day collide. + # A later same-day run that emits no cdna row can still bind the earlier run's stale + # transcript. Only a versioned run anchor closes this fully. cdna_transcript_by_run: dict[tuple[int, date | None], str | None] = { (target_gene_id, mapped_date): reference_accession for target_gene_id, mapped_date, reference_accession in job_manager.db.execute( @@ -243,10 +213,8 @@ async def reverse_translate_variants_for_score_set( .all() } - # Load current, authoritative, and successfully-mapped MappingRecords along with their - # target gene (for the coding-transcript lookup) and parent Variant. - # The joined TargetGeneMapping is the record's own (genomic) mapping row, so its - # mapped_date is this record's run date -- used to anchor the cdna lookup to the run. + # Load current authoritative MappingRecords with their Variant and TargetGeneMapping. + # mapped_date from the joined (genomic) TargetGeneMapping anchors the cdna transcript lookup. rows: Sequence[tuple[MappingRecord, Variant, int, date | None]] = ( job_manager.db.execute( select( @@ -275,10 +243,8 @@ async def reverse_translate_variants_for_score_set( job_manager.db.flush() return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) - # Resolve each record's coding transcript. Genomic/cdna mappings share their target - # gene's cdna alignment reference; protein mappings have none (the mapper emits - # reference_accession only for cdna alignments), so gather their protein accessions and - # resolve NP_→NM_ from UTA in a single batch query below. + # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no + # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. transcript_resolutions: list[_TranscriptResolution] = [] protein_accessions: set[str] = set() for rec, variant, target_gene_id, mapped_date in rows: @@ -296,9 +262,8 @@ async def reverse_translate_variants_for_score_set( transcript_by_protein = _coding_transcripts_for_proteins(protein_accessions) - # Target gene category per gene, so a skip can be classified honestly: a coding target - # with no resolvable transcript is recoverable (should now be rare), while a regulatory / - # non-coding target has no protein consequence and is a correct skip. + # Target category per gene — used to classify skips as recoverable (coding target, + # transcript unresolved) vs. correct (non-coding/regulatory, no protein consequence). target_category_by_gene: dict[int, TargetCategory] = dict( job_manager.db.execute( select(TargetGene.id, TargetGene.category).where(TargetGene.score_set_id == score_set_id) @@ -307,14 +272,9 @@ async def reverse_translate_variants_for_score_set( .all() ) - # Build VariantInputs, supplying the resolved coding transcript for every input - # regardless of its own assay level (p./c./g. all collapse to a ProteinConsequence - # anchored on that transcript). Object identity on each VariantInput is preserved - # through the call so we can correlate TranslationResult.input back to its MappingRecord. - # - # A record with no coding transcript (e.g. a regulatory element aligned only at the - # genomic level, or a protein with no UTA association) has no protein consequence to - # reverse-translate; it is skipped and recorded as SKIPPED rather than counted as a failure. + # Build VariantInputs with the resolved coding transcript (p./c./g. all collapse to a + # ProteinConsequence on that transcript). Object identity is preserved so TranslationResult + # can be correlated back to its MappingRecord. Records with no transcript are skipped. variant_inputs: list[Any] = [] variant_input_map: dict[int, tuple[MappingRecord, Variant]] = {} skipped_variants: list[_TranscriptResolution] = [] @@ -378,9 +338,8 @@ async def reverse_translate_variants_for_score_set( .where(MappingRecord.current.is_(True)) ) - # Collect new MappingRecord -> Allele links and defer linkage until all candidates are processed. - # This allows us to supersede the prior live derived links with the new set in one atomic operation at the end, - # so the retire and insert share a timestamp and there is no gap between the old and new sets of live links. + # Defer linkage until all candidates are processed so the prior derived links can be + # superseded atomically — retire and insert share a timestamp with no gap. new_links: list[MappingRecordAllele] = [] # The live authoritative (record, allele) pairs. A derived candidate that equals a record's @@ -410,12 +369,15 @@ async def reverse_translate_variants_for_score_set( (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] + # Emit the protein consequence as the protein-level member of the equivalence set. + # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. + # None for protein-assay inputs where the protein is already the authoritative allele. + if result.hgvs_p: + candidates.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p")) + for hgvs, level, hgvs_field in candidates: - # A candidate the equivalence class produced may be a form ga4gh cannot - # translate (an intronic projection, an unsupported edge case, a malformed - # bracketed expression). A variant with at least one translatable candidate still - # advances the pipeline; only a variant where every candidate fails is - # recorded as failed (with the per-candidate errors retained as metadata). + # A candidate may not be translatable (intronic projection, malformed expression). + # Track per-candidate failures; a variant fails only if *all* candidates fail. try: variation = translate_hgvs_to_variation(hgvs, allele_translator) except Exception as e: @@ -483,10 +445,9 @@ async def reverse_translate_variants_for_score_set( annotation_data={"annotation_metadata": annotation_metadata}, ) - # Supersede the prior live derived links with the new set in one gap-free operation. - # TODO#765: a re-run supersedes the whole derived set wholesale because re-mapping re-mints the - # records, so we cannot tell which derivations are unchanged; idempotent mapping records would let - # unchanged derived links stay live instead of being retired and recreated. + # Supersede prior live derived links atomically. + # TODO#765: re-runs retire and recreate the whole derived set because re-mapping re-mints + # records; idempotent records would allow unchanged links to stay live. MappingRecordAllele.supersede_live_where( job_manager.db, new_links, diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 459ec53db..ef8104b94 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -69,12 +69,15 @@ def _construct(inputs, *, transcripts, coordinates, config): if inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: - c_candidates, g_candidates = results_by_hgvs[inp.hgvs] + entry = results_by_hgvs[inp.hgvs] + c_candidates, g_candidates = entry[0], entry[1] + hgvs_p = entry[2] if len(entry) > 2 else None results.append( TranslationResult( input=inp, hgvs_c_candidates=list(c_candidates), hgvs_g_candidates=list(g_candidates), + hgvs_p=hgvs_p, ) ) else: @@ -323,6 +326,57 @@ async def test_transcript_is_queryable_via_derived_expression( coding = session.query(Allele).filter(Allele.transcript == "NM_000001.1").all() assert [a.vrs_digest for a in coding] == ["ga4gh:VA.coding"] + async def test_protein_consequence_is_persisted_as_a_protein_allele( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The deterministic protein consequence (``result.hgvs_p``) is emitted as the + protein-level member of the equivalence set -- a non-authoritative ``level=protein`` + allele linked to the record, not dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + # c_to_p emits a predicted consequence in parens; the job strips them before use. + p_consequence = "NP_000001.1:p.(Met1Val)" + p_stripped = "NP_000001.1:p.Met1Val" + construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate], p_consequence)}) + translate = fake_translate( + { + g_candidate: "ga4gh:VA.genomic", + c_candidate: "ga4gh:VA.coding", + p_stripped: "ga4gh:VA.protein", # the stripped form is what reaches the translator + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() + assert protein_allele.level == AnnotationLayer.protein.value + assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens + # Linked to the record as a non-authoritative member of the equivalence set. + assert protein_allele.id in {link.allele_id for link in _non_authoritative_links(session)} + async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( self, session, From 08634e45e251afbeefb813c0e1da7f29cf8cd8be Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 15:18:22 -0700 Subject: [PATCH 19/93] feat(mapping): use typed MappingOutcome to distinguish benign skips from failures Previously, variant mapping success was determined solely by the presence of pre/post-mapped alleles. This conflated genuinely failed mappings with benign absences (intronic variants, no-protein-consequence variants) that legitimately produce no allele. - Add `MappingOutcome` enum to `lib/mapping/schema.py` mirroring `dcd_mapping.schemas.MappingOutcome`, with an `is_benign_absence` helper to classify INTRONIC and NO_PROTEIN_CONSEQUENCE outcomes - Rewrite the per-record outcome logic in `map_variants_for_score_set` to branch on the typed outcome rather than allele presence: MAPPED -> SUCCESS, benign absence -> SKIPPED, FAILED -> FAILED - Replace the `successful_mapped_variants` scalar with a typed `Counter[MappingOutcome]` that feeds `mapped_count`, `failed_count`, and `skipped_count` tallies in logs and final state decision - Derive `mapping_state` from genuine failures only; all-benign result sets are treated as complete, not failed - Raise `NonexistentMappingResultsError` when a score annotation has no `outcome` field (malformed/older payload) - Expand test coverage for intronic and no-protein-consequence scenarios, verifying correct status and failure-category assignment --- src/mavedb/lib/mapping/schema.py | 22 +++++ .../worker/jobs/variant_processing/mapping.py | 83 ++++++++++++----- tests/helpers/util/setup/worker.py | 6 ++ .../jobs/variant_processing/test_mapping.py | 90 +++++++++++++++++++ 4 files changed, 180 insertions(+), 21 deletions(-) diff --git a/src/mavedb/lib/mapping/schema.py b/src/mavedb/lib/mapping/schema.py index 939b60401..02d377780 100644 --- a/src/mavedb/lib/mapping/schema.py +++ b/src/mavedb/lib/mapping/schema.py @@ -7,9 +7,31 @@ """ from datetime import datetime +from enum import Enum from typing import Any, Optional, TypedDict +class MappingOutcome(str, Enum): + """Per-record outcome stamped by dcd-mapping on every emitted score annotation. + + Mirrors ``dcd_mapping.schemas.MappingOutcome``. Lets a benign absence of a VRS allele + be told apart from a genuine failure -- a distinction ``error_message`` alone cannot + make (benign outcomes leave it ``None``). ``MAPPED`` is a success; ``INTRONIC`` and + ``NO_PROTEIN_CONSEQUENCE`` are benign skips (no allele, not failures); ``FAILED`` is the + only genuine failure. + """ + + MAPPED = "mapped" + INTRONIC = "intronic" + NO_PROTEIN_CONSEQUENCE = "no_protein_consequence" + FAILED = "failed" + + @property + def is_benign_absence(self) -> bool: + """True for outcomes that legitimately produce no allele yet are not failures.""" + return self in (MappingOutcome.INTRONIC, MappingOutcome.NO_PROTEIN_CONSEQUENCE) + + class GeneInfo(TypedDict, total=False): hgnc_symbol: Optional[str] selection_method: Optional[str] diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index d8b02cebb..9e6a8da74 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -8,6 +8,7 @@ import asyncio import functools import logging +from collections import Counter from datetime import date from typing import Any @@ -23,6 +24,7 @@ ) from mavedb.lib.logging.context import format_raised_exception_info_as_dict from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS +from mavedb.lib.mapping.schema import MappingOutcome from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped @@ -258,7 +260,10 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan total_variants = len(mapped_scores) job_manager.save_to_context({"total_variants_to_process": total_variants}) - successful_mapped_variants = 0 + # Tally every record by its typed outcome; the mapped/failed/benign buckets are + # derived from this after the loop. Keeping all four keys preserves the + # intronic-vs-no-protein distinction in logs. + outcome_counts: Counter[MappingOutcome] = Counter() logger.info( f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), @@ -282,10 +287,17 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan if existing_mapped_variant: job_manager.save_to_context({"existing_mapped_variant": existing_mapped_variant.id}) - annotation_was_successful = mapped_score.get("pre_mapped") and mapped_score.get("post_mapped") - if annotation_was_successful: - successful_mapped_variants += 1 - job_manager.save_to_context({"successful_mapped_variants": successful_mapped_variants}) + # The typed outcome -- not allele presence -- decides success/benign/failure. + # Absent outcome means an older or malformed payload; fail fast. + raw_outcome = mapped_score.get("outcome") + if not raw_outcome: + raise NonexistentMappingResultsError( + f"ScoreAnnotation for variant {variant_urn!r} is missing its outcome." + ) + outcome = MappingOutcome(raw_outcome) + + outcome_counts[outcome] += 1 + job_manager.save_to_context({"outcome_counts": {o.value: n for o, n in outcome_counts.items()}}) # dcd-mapping guarantees both fields are set on every ScoreAnnotation, # including failed variants (the annotate step re-attributes failures to @@ -337,25 +349,36 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan else: job_manager.db.add(mapping_record) + # MAPPED -> success; benign absences -> skipped; FAILED -> failed. The raw outcome + # is preserved in annotation_metadata so the benign distinction survives. + if outcome is MappingOutcome.MAPPED: + annotation_status = AnnotationStatus.SUCCESS + annotation_failure_category = None + elif outcome.is_benign_absence: + annotation_status = AnnotationStatus.SKIPPED + annotation_failure_category = None + else: + annotation_status = AnnotationStatus.FAILED + annotation_failure_category = AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED + annotation_manager.add_annotation( variant_id=variant.id, # type: ignore annotation_type=AnnotationType.VRS_MAPPING, version=tool_version, - status=AnnotationStatus.SUCCESS if annotation_was_successful else AnnotationStatus.FAILED, - failure_category=None - if annotation_was_successful - else AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + status=annotation_status, + failure_category=annotation_failure_category, annotation_data={ "error_message": mapped_score.get("error_message", null()), "annotation_metadata": { + "outcome": outcome.value, "mapped_assay_level_hgvs": assay_level_hgvs, }, }, current=True, ) - # Only variants with a post-mapped representation yield an authoritative Allele. Failed variants - # get a MappingRecord (with null VRS data) but no linked allele. + # Only variants with a post-mapped representation yield an authoritative Allele; + # failed and benign-absent variants get a MappingRecord but no linked allele. if post_mapped_allele: allele_draft = AlleleDbModel( vrs_digest=post_mapped_allele["id"], @@ -383,17 +406,26 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan annotation_manager.flush() - if successful_mapped_variants == 0: + # Collapse the per-outcome tally into the three buckets the rest of the job reasons about. + mapped_count = outcome_counts[MappingOutcome.MAPPED] + failed_count = outcome_counts[MappingOutcome.FAILED] + skipped_count = sum(n for o, n in outcome_counts.items() if o.is_benign_absence) + + # State keys off genuine failures only: no failures -> complete (benign absences + # don't count); failures with nothing mapped -> failed; otherwise incomplete. + if failed_count == 0: + score_set.mapping_state = MappingState.complete + elif mapped_count == 0: score_set.mapping_state = MappingState.failed score_set.mapping_errors = {"error_message": "All variants failed to map."} - elif successful_mapped_variants < total_variants: - score_set.mapping_state = MappingState.incomplete else: - score_set.mapping_state = MappingState.complete + score_set.mapping_state = MappingState.incomplete job_manager.save_to_context( { - "successful_mapped_variants": successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "mapping_state": score_set.mapping_state.name, "mapping_errors": score_set.mapping_errors, "inserted_mapped_variants": len(mapped_scores), @@ -442,7 +474,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan logger.info(msg="Inserted mapped variants into db.", extra=job_manager.logging_context()) - if successful_mapped_variants == 0: + # Fail the job only on genuine failure with nothing mapped; all-benign is a success. + if mapped_count == 0 and failed_count > 0: logger.error(msg="No variants were successfully mapped.", extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -450,19 +483,27 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan data={ "score_set_id": score_set.id, "mapped_count": 0, - "unmapped_count": total_variants, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, }, failure_category=FailureCategory.VRS_MAPPING_FAILED, ) - logger.info(msg="Variant mapping job completed successfully.", extra=job_manager.logging_context()) + logger.info( + msg=( + f"Variant mapping job completed successfully: {mapped_count} mapped, " + f"{failed_count} failed, {skipped_count} skipped (intronic / no protein consequence)." + ), + extra=job_manager.logging_context(), + ) job_manager.db.flush() return JobExecutionOutcome.succeeded( data={ "score_set_id": score_set.id, - "mapped_count": successful_mapped_variants, - "unmapped_count": total_variants - successful_mapped_variants, + "mapped_count": mapped_count, + "failed_count": failed_count, + "skipped_count": skipped_count, "total_count": total_variants, } ) diff --git a/tests/helpers/util/setup/worker.py b/tests/helpers/util/setup/worker.py index 815dd1a90..252471ca8 100644 --- a/tests/helpers/util/setup/worker.py +++ b/tests/helpers/util/setup/worker.py @@ -183,6 +183,12 @@ async def construct_mock_mapping_output( if not with_all_variants and idx % 2 == 0: mapped_score["post_mapped"] = {} + # Mirror the mapper's per-record outcome: both alleles present -> MAPPED, else + # FAILED. Tests needing benign outcomes set ``outcome`` explicitly afterward. + mapped_score["outcome"] = ( + "mapped" if mapped_score["pre_mapped"] and mapped_score["post_mapped"] else "failed" + ) + mapping_output["mapped_scores"].append(mapped_score) if not mapping_output["mapped_scores"]: diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 74c9acb1c..3a0c6db2f 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -684,6 +684,96 @@ async def dummy_mapping_job(): assert len(annotation_status_failed) == 1 assert annotation_status_failed[0].annotation_type == "vrs_mapping" + async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( + self, + session, + with_independent_processing_runs, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_score_set, + ): + """A score set whose only unmapped variants are benign absences (intronic / no + protein consequence) is ``complete``, not ``incomplete``/``failed``: benign + outcomes carry no allele but are skips, not failures.""" + + async def dummy_mapping_job(): + mapping_output = await construct_mock_mapping_output( + session=session, + score_set=sample_score_set, + with_gene_info=True, + with_layers={"g", "c", "p"}, + with_pre_mapped=True, + with_post_mapped=True, + with_reference_metadata=True, + with_mapped_scores=True, + with_all_variants=True, + ) + # Re-stamp the emitted records as benign absences: no allele, no failure. The + # helper only produces MAPPED/FAILED, so we override here to exercise the path. + benign_outcomes = ["intronic", "no_protein_consequence"] + for idx, mapped_score in enumerate(mapping_output["mapped_scores"]): + mapped_score["pre_mapped"] = {} + mapped_score["post_mapped"] = {} + mapped_score["outcome"] = benign_outcomes[idx % len(benign_outcomes)] + return mapping_output + + variant1 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + urn="variant:1", + ) + variant2 = Variant( + score_set_id=sample_score_set.id, + hgvs_nt="NM_000000.1:c.2G>T", + hgvs_pro="NP_000000.1:p.Val2Leu", + data={}, + urn="variant:2", + ) + session.add_all([variant1, variant2]) + session.commit() + + with ( + patch.object( + _UnixSelectorEventLoop, + "run_in_executor", + return_value=dummy_mapping_job(), + ), + ): + result = await map_variants_for_score_set( + mock_worker_ctx, + sample_independent_variant_mapping_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_independent_variant_mapping_run.id), + ) + + # The run succeeded and the score set is complete -- nothing genuinely failed. + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.SUCCEEDED + assert result.data["mapped_count"] == 0 + assert result.data["failed_count"] == 0 + assert result.data["skipped_count"] == 2 + assert sample_score_set.mapping_state == MappingState.complete + assert sample_score_set.mapping_errors is None + + # A record exists per variant, but with no authoritative allele. + mapping_records = session.query(MappingRecord).all() + assert len(mapping_records) == 2 + assert all(authoritative_allele_for(session, r) is None for r in mapping_records) + + # Benign outcomes are SKIPPED (not FAILED), and the finer outcome is preserved in metadata. + annotation_statuses = ( + session.query(VariantAnnotationStatus) + .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id) + .all() + ) + assert len(annotation_statuses) == 2 + assert all(s.status == "skipped" for s in annotation_statuses) + assert all(s.failure_category is None for s in annotation_statuses) + recorded_outcomes = {s.annotation_metadata["outcome"] for s in annotation_statuses} + assert recorded_outcomes == {"intronic", "no_protein_consequence"} + async def test_map_variants_for_score_set_complete_mapping( self, session, From 0f1ea170f263eb3ec2db89974d6b2b0f74b76594 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 9 Jun 2026 22:55:11 -0700 Subject: [PATCH 20/93] refactor(reverse-translation): replace NullTranscriptSource with live UTA-backed source WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, which requires a real UTA connection. The previous NullTranscriptSource always returned None, silently breaking WT-codon resolution. - Extract `uta_transcript_source()` context manager into `translation_ports.py`; removes the ad-hoc UTA connection setup that was duplicated in the job and the now-deleted NullTranscriptSource - Scope the live UTA client around the full `run_in_executor` call so the connection outlives the synchronous executor block - Remove NullTranscriptSource; callers that only need transcript_for_protein (already resolved by the job) also benefit from the real client without extra cost - Add a TODO noting that non-substitution consequences (del/ins/delins/ fs/ext) are miscounted as FAILED rather than SKIPPED (#767) --- .../variant_processing/reverse_translation.py | 43 ++++++++++--------- src/mavedb/worker/lib/translation_ports.py | 37 ++++++++++------ .../test_reverse_translation.py | 17 +++++--- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 872044e91..1bf918b0a 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -8,11 +8,9 @@ """ import asyncio -import contextlib import dataclasses import functools import logging -import os from datetime import date from enum import Enum from typing import Any, NamedTuple, Sequence @@ -20,7 +18,6 @@ from ga4gh.vrs.extras.translator import AlleleTranslator from sqlalchemy import select from variant_annotation.lib.accessions import looks_like_refseq_protein_accession -from variant_annotation.lib.clients.uta import UtaClient, connect_uta from variant_annotation.lib.translation import construct_equivalent_variants from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode @@ -43,7 +40,7 @@ from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager -from mavedb.worker.lib.translation_ports import NullTranscriptSource, WorkerCoordinateTranslator +from mavedb.worker.lib.translation_ports import WorkerCoordinateTranslator, uta_transcript_source logger = logging.getLogger(__name__) @@ -109,12 +106,7 @@ def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, if not protein_accessions: return {} - uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() - if not uta_db_url: - raise RuntimeError("UTA_DB_URL must be set to resolve protein→transcript associations.") - - with contextlib.closing(connect_uta(uta_db_url)) as conn: - client = UtaClient(conn) + with uta_transcript_source() as client: return { pro_ac: transcript for pro_ac in sorted(protein_accessions) @@ -305,19 +297,24 @@ async def reverse_translate_variants_for_score_set( # construct_equivalent_variants is I/O-bound (blocks on a subprocess); run in the # default thread pool rather than the process pool to avoid pickling the port objects. coordinates = WorkerCoordinateTranslator(ctx["hdp"]) - transcripts = NullTranscriptSource() loop = asyncio.get_running_loop() job_manager.update_progress(20, 100, "Running reverse translation subprocess.") - results, errors = await loop.run_in_executor( - None, - functools.partial( - construct_equivalent_variants, - variant_inputs, - transcripts=transcripts, - coordinates=coordinates, - config=translation_config, - ), - ) + # WtCodonMode.ALL reads the reference codon via TranscriptSource.codon_at, so pass a + # live UTA-backed source. codon_at is only touched in the post-subprocess WT-codon step, + # but the connection must outlive the executor call, so scope it around the whole call. + # (transcript_for_protein is redundant here -- the job already resolved every transcript + # and supplies it via VariantInput.transcript.) + with uta_transcript_source() as transcripts: + results, errors = await loop.run_in_executor( + None, + functools.partial( + construct_equivalent_variants, + variant_inputs, + transcripts=transcripts, + coordinates=coordinates, + config=translation_config, + ), + ) job_manager.update_progress(70, 100, "Writing translated alleles to database.") logger.info( @@ -455,6 +452,10 @@ async def reverse_translate_variants_for_score_set( MappingRecordAllele.mapping_record_id.in_(current_record_ids), ) + # TODO#767: non-substitution consequences (del/ins/delins/fs/ext) have no synonymous + # equivalence class, so they arrive here as TranslationErrors and are miscounted as + # FAILED. Classify by the protein consequence's edit type up front and map these to + # SKIPPED instead of pattern-matching engine error strings. for error in errors: _rec, variant = variant_input_map[id(error.input)] failed += 1 diff --git a/src/mavedb/worker/lib/translation_ports.py b/src/mavedb/worker/lib/translation_ports.py index 467d2f93a..b715aace4 100644 --- a/src/mavedb/worker/lib/translation_ports.py +++ b/src/mavedb/worker/lib/translation_ports.py @@ -1,8 +1,9 @@ """Worker-side adapters for the variant_annotation translation ports. construct_equivalent_variants depends on two protocols — CoordinateTranslator -and TranscriptSource. These are the worker's concrete implementations, so jobs -can pass them in without importing the port definitions directly. +and TranscriptSource. This module supplies the worker's CoordinateTranslator +(WorkerCoordinateTranslator) and a factory (uta_transcript_source) for the +TranscriptSource, which is variant_annotation's own UTA-backed UtaClient. WorkerCoordinateTranslator defers AssemblyMapper initialization until the first call — the hgvs library makes network calls on mapper construction that must not @@ -11,8 +12,28 @@ from __future__ import annotations +import contextlib +import os +from collections.abc import Generator from typing import Any +from variant_annotation.lib.clients.uta import UtaClient, connect_uta + + +@contextlib.contextmanager +def uta_transcript_source() -> Generator[UtaClient]: + """Yield a UTA-backed TranscriptSource over a connection scoped to the block. + + Backs both the NP_→NM_ association lookup and the WT-codon read used by + WtCodonMode.ALL (TranscriptSource.codon_at). The connection is closed on exit, + so callers must use the client within the ``with`` block. + """ + uta_db_url = (os.environ.get("UTA_DB_URL") or "").strip() + if not uta_db_url: + raise RuntimeError("UTA_DB_URL must be set to resolve transcript facts (NP_→NM_ associations and WT codons).") + with contextlib.closing(connect_uta(uta_db_url)) as conn: + yield UtaClient(conn) + class WorkerCoordinateTranslator: """CoordinateTranslator backed by the worker's HGVS data provider.""" @@ -45,15 +66,3 @@ def g_to_c(self, g_hgvs: str, transcript: str) -> str: def c_to_g(self, c_hgvs: str) -> str: self._ensure_initialized() return str(self._mapper.c_to_g(self._parser.parse(c_hgvs))) - - -class NullTranscriptSource: - """Null TranscriptSource — the reverse-translation job resolves every coding transcript - itself (including the UTA NP_→NM_ lookup) and always supplies it via VariantInput.transcript, - so the library never needs to resolve one.""" - - def transcript_for_protein(self, protein_accession: str) -> str | None: - return None - - def codon_at(self, transcript: str, aa_position: int) -> str | None: - return None diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index ef8104b94..ec926f267 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -4,9 +4,10 @@ pytest.importorskip("arq") +import contextlib from asyncio.unix_events import _UnixSelectorEventLoop from datetime import timedelta -from unittest.mock import patch +from unittest.mock import MagicMock, patch from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode @@ -130,11 +131,15 @@ async def dummy_mapping_job(): async def _reverse_translate(session, mock_worker_ctx, rt_run): - return await reverse_translate_variants_for_score_set( - mock_worker_ctx, - rt_run.id, - JobManager(session, mock_worker_ctx["redis"], rt_run.id), - ) + # The job opens a UTA-backed TranscriptSource to back codon_at (WtCodonMode.ALL). + # construct_equivalent_variants is mocked in these tests, so the source is never + # queried -- stub the factory with a no-op context yielding a dummy client. + with patch(f"{RT_MODULE}.uta_transcript_source", lambda: contextlib.nullcontext(MagicMock())): + return await reverse_translate_variants_for_score_set( + mock_worker_ctx, + rt_run.id, + JobManager(session, mock_worker_ctx["redis"], rt_run.id), + ) def _cross_level_statuses(session, score_set_id, status=None): From 6e45f562f61e1aacc3a9f5ad2524be02ab5d121d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 12:33:21 -0700 Subject: [PATCH 21/93] fix(migration): correct down_revision reference in allele closure tables migration --- .../versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py index 49c502d05..6c1fc1d99 100644 --- a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py +++ b/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py @@ -1,7 +1,7 @@ """Add mapping_records, alleles, and mapping_record_alleles tables Revision ID: a1b2c3d4e5f6 -Revises: 398067c53257 +Revises: a7f3c2e9b104 Create Date: 2026-05-29 New parallel tables for the Better Reverse Translation epic (#746). @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "a1b2c3d4e5f6" -down_revision = "398067c53257" +down_revision = "a7f3c2e9b104" branch_labels = None depends_on = None From d3f4da9fc02bdd9ab579164631d5538c43299f55 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 18 Jun 2026 15:07:06 -0700 Subject: [PATCH 22/93] fix(vrs): stamp HGVS expressions on reverse-translated alleles and handle null gracefully - `translate_hgvs_to_variation` now attaches an `Expression` to each allele produced from a cis-phased or single HGVS string, mirroring the dcd_mapping authoritative-allele convention so `post_mapped` is self-describing without a separate round-trip. - `hgvs_from_vrs_allele` returns `None` instead of crashing when `expressions` is null or empty (valid for cis-phased block members), and `get_hgvs_from_post_mapped` propagates that as a `None` result. - `post_mapped` is now serialized with `exclude_none=True`, matching the mapper's output format. - Minor formatting clean-ups in reverse_translation.py (no logic change). --- src/mavedb/lib/variants.py | 37 +++++++++++-------- src/mavedb/lib/vrs_utils.py | 33 +++++++++++++++-- .../variant_processing/reverse_translation.py | 3 +- tests/lib/test_variants.py | 20 ++++++++++ .../test_reverse_translation.py | 3 +- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index a941006cf..c2d5595f9 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -10,20 +10,23 @@ HGVS_P_REGEX = re.compile(r"(^|:)p\.") -def hgvs_from_vrs_allele(allele: dict) -> str: +def hgvs_from_vrs_allele(allele: dict) -> Optional[str]: """ - Extract the HGVS notation from the VRS allele. + Extract the HGVS notation from the VRS allele, or None if it carries no expression. """ try: - # VRS 2.X - return allele["expressions"][0]["value"] + expressions = allele["expressions"] # VRS 2.X except KeyError: if "variation" in allele: raise ValueError("VRS 1.X format not supported.") # VRS 1.X. We don't want to allow this. - # return allele["variation"]["expressions"][0]["value"] - else: - raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + raise KeyError("Invalid VRS allele structure. Expected 'expressions'.") + + # A valid VRS allele may simply carry no HGVS expression (None or empty) — e.g. a member of a + # cis-phased block. That is "no HGVS", not a crash. + if not expressions: + return None + return expressions[0]["value"] def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bool = False) -> Optional[str]: @@ -38,22 +41,24 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bo if not post_mapped_vrs: return None - if post_mapped_vrs["type"] == "Haplotype": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] - elif post_mapped_vrs["type"] == "CisPhasedBlock": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(allele) for allele in post_mapped_vrs["members"]] + if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore + members = post_mapped_vrs["members"] elif post_mapped_vrs["type"] == "Allele": # type: ignore - variations_hgvs = [hgvs_from_vrs_allele(post_mapped_vrs)] + members = [post_mapped_vrs] else: return None - if len(variations_hgvs) == 0: + member_hgvs = [hgvs_from_vrs_allele(allele) for allele in members] + + # No members, or a member carrying no HGVS expression — no single/combinable HGVS to return. + if not member_hgvs or any(h is None for h in member_hgvs): return None - if len(variations_hgvs) > 1: - return join_cis_phased_hgvs(variations_hgvs) if combine_cis else None + hgvs_values: list[str] = [h for h in member_hgvs if h is not None] + if len(hgvs_values) > 1: + return join_cis_phased_hgvs(hgvs_values) if combine_cis else None - return variations_hgvs[0] + return hgvs_values[0] def get_digest_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index 052c4b702..d5a6432ee 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -17,14 +17,35 @@ from ga4gh.vrs.models import ( Allele, CisPhasedBlock, + Expression, LiteralSequenceExpression, ReferenceLengthExpression, SequenceLocation, + Syntax, ) from ga4gh.vrs.normalize import normalize from mavedb.lib.hgvs import split_cis_phased_hgvs +# HGVS type letter (``accession:g.``) → VRS Expression syntax. +_HGVS_SYNTAX_BY_TYPE = { + "g": Syntax.HGVS_G, + "c": Syntax.HGVS_C, + "p": Syntax.HGVS_P, + "n": Syntax.HGVS_N, + "m": Syntax.HGVS_M, + "r": Syntax.HGVS_R, +} + + +def _hgvs_syntax(hgvs: str) -> Syntax: + """Map an HGVS string to its VRS Expression syntax via the type letter after the accession.""" + _, _, rest = hgvs.partition(":") + try: + return _HGVS_SYNTAX_BY_TYPE[rest[:1]] + except KeyError: + raise ValueError(f"Cannot determine HGVS syntax for {hgvs!r}") + def translate_hgvs_to_vrs(hgvs: str, translator: AlleleTranslator) -> Allele: """Convert HGVS variation description to VRS object. @@ -81,10 +102,14 @@ def translate_hgvs_to_variation(hgvs: str, translator: AlleleTranslator) -> Alle :param translator: caller-owned AlleleTranslator reused across calls :return: an Allele for a single variant, or a CisPhasedBlock for a cis-phased set """ - members = [ - normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) - for component in split_cis_phased_hgvs(hgvs) - ] + members = [] + for component in split_cis_phased_hgvs(hgvs): + allele = normalize_and_identify(translate_hgvs_to_vrs(component, translator), translator.data_proxy) + # Stamp the source HGVS as the allele's expression so post_mapped is self-describing. + # Mirrors the mapper's authoritative alleles. + allele.expressions = [Expression(syntax=_hgvs_syntax(component), value=component)] + members.append(allele) + if len(members) == 1: return members[0] diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 1bf918b0a..07df529fd 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -391,7 +391,8 @@ async def reverse_translate_variants_for_score_set( seen_digests.add(variation.id) draft_allele = AlleleDbModel( vrs_digest=variation.id, - post_mapped=variation.model_dump(), + # exclude_none mirrors the mapper's serialization. + post_mapped=variation.model_dump(exclude_none=True), level=level, **{hgvs_field: hgvs}, # type: ignore[arg-type] ) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index 9cffaa793..8b25197ca 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -83,6 +83,26 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) +def test_hgvs_from_vrs_allele_null_or_empty_expressions(): + # A VRS allele may carry `expressions: null` or `[]` — that is "no HGVS", not a crash. + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": None}) is None + assert hgvs_from_vrs_allele({"type": "Allele", "expressions": []}) is None + + +def test_get_hgvs_from_post_mapped_member_without_expression(): + # Regression: a cis-phased block member whose `expressions` is null must yield None, not raise + # `TypeError: 'NoneType' object is not subscriptable` (which previously killed the CAR job). + block = { + "type": "CisPhasedBlock", + "members": [ + {"type": "Allele", "expressions": [{"value": "NM_003345:p.Asp5Phe"}]}, + {"type": "Allele", "expressions": None}, + ], + } + assert get_hgvs_from_post_mapped(block) is None + assert get_hgvs_from_post_mapped(block, combine_cis=True) is None + + ### Tests for get_digest_from_post_mapped function ### diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index ec926f267..970fa8544 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -50,7 +50,8 @@ def __init__(self, vrs_id: str, vrs_type: str = "Allele"): self.id = vrs_id self.vrs_type = vrs_type - def model_dump(self) -> dict: + def model_dump(self, **kwargs) -> dict: + # Accept (and ignore) model_dump kwargs like exclude_none, mirroring the real VRS model. return {"type": self.vrs_type, "id": self.id} From 225f824f01aa665d2fe15fe77c325f847d8b410f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 18 Jun 2026 15:10:28 -0700 Subject: [PATCH 23/93] feat(clingen): migrate CAR/LDH jobs from MappedVariant to Allele data model - `submit_score_set_mappings_to_car` now operates on Allele rows (authoritative + RT-derived) rather than MappedVariant, deduplicating by allele_id so each VRS allele is registered exactly once regardless of how many variants share it. Adds `force_reregister` param and per-allele outcome counters. - `submit_score_set_mappings_to_ldh` queries MappingRecord + Allele for pre/post-mapped data instead of the deprecated MappedVariant join. - `construct_ldh_submission_entity` signature updated to accept MappingRecord and Allele separately, since those fields now live on different models. - `warm_clingen_cache` switched to the shared `get_alleles_for_score_set` helper to keep allele scope consistent across all three jobs. - Extracts `get_alleles_for_score_set` and `ScoreSetAlleleRow` into `lib/clingen/alleles.py` as the single canonical query for both CAR and cache jobs. --- src/mavedb/lib/clingen/alleles.py | 61 ++ .../lib/clingen/content_constructors.py | 26 +- src/mavedb/models/score_set.py | 2 +- .../worker/jobs/external_services/clingen.py | 481 +++++++---- .../jobs/external_services/clingen_cache.py | 26 +- .../lib/clingen/test_content_constructors.py | 27 +- tests/lib/conftest.py | 17 + .../jobs/external_services/test_clingen.py | 771 ++++++++++++------ .../external_services/test_clingen_cache.py | 185 +++-- 9 files changed, 1100 insertions(+), 496 deletions(-) create mode 100644 src/mavedb/lib/clingen/alleles.py diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py new file mode 100644 index 000000000..527572233 --- /dev/null +++ b/src/mavedb/lib/clingen/alleles.py @@ -0,0 +1,61 @@ +"""Query helpers for fetching score-set alleles for ClinGen registration. + +Both submit_score_set_mappings_to_car and warm_clingen_cache use the same allele +scope: all current MappingRecordAllele links (authoritative and RT-derived) for a +score set. A single definition here prevents the two jobs from drifting apart. +""" + +from typing import NamedTuple + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + + +class ScoreSetAlleleRow(NamedTuple): + """One (allele, variant) link for a score set. An allele shared by multiple variants + appears once per variant so callers can fan annotation statuses out correctly. + + ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be + the authoritative measurement for one variant and an RT-derived equivalence for another. + """ + + allele_id: int + post_mapped: dict | None + clingen_allele_id: str | None + variant_id: int + is_authoritative: bool + + +def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: + """Return all current alleles for a score set with their linked variant IDs. + + Covers both authoritative mapper alleles and RT-derived equivalence alleles — + the full set that requires ClinGen registration before the annotation fan-out + can run. + + Only alleles with a non-null ``post_mapped`` are returned — variants that failed + or were benignly absent have no allele link and cannot receive a CAID. + """ + rows = db.execute( + select( + Allele.id, + Allele.post_mapped, + Allele.clingen_allele_id, + Variant.id.label("variant_id"), + MappingRecordAllele.is_authoritative, + ) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == score_set_id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(Allele.post_mapped.is_not(None)) + ).all() + + return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] diff --git a/src/mavedb/lib/clingen/content_constructors.py b/src/mavedb/lib/clingen/content_constructors.py index 1f55437f4..ce62ebbbd 100644 --- a/src/mavedb/lib/clingen/content_constructors.py +++ b/src/mavedb/lib/clingen/content_constructors.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from uuid import uuid4 from urllib.parse import quote_plus @@ -7,7 +6,8 @@ from mavedb.constants import MAVEDB_BASE_GIT, MAVEDB_FRONTEND_URL from mavedb.lib.types.clingen import LdhContentLinkedData, LdhContentSubject, LdhEvent, LdhSubmission from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.variant import Variant @@ -32,8 +32,12 @@ def construct_ldh_submission_subject(hgvs: str) -> LdhContentSubject: return {"Variant": {"hgvs": hgvs}} -def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[MappedVariant]) -> LdhContentLinkedData: - entity: LdhContentLinkedData = { +def construct_ldh_submission_entity( + variant: Variant, mapping_record: MappingRecord, allele: Allele +) -> LdhContentLinkedData: + # Pre-mapped data and the mapping API version live on the per-variant MappingRecord; + # post-mapped data lives on the (cross-variant deduped) Allele. + return { # TODO#372: We try to make all possible fields that are non-nullable represented that way. "MaveDBMapping": [ { @@ -41,27 +45,25 @@ def construct_ldh_submission_entity(variant: Variant, mapped_variant: Optional[M "mavedb_id": variant.urn, # type: ignore "score": variant.data["score_data"]["score"], # type: ignore "score_set_description": variant.score_set.short_description, # type: ignore + "pre_mapped": mapping_record.pre_mapped, + "post_mapped": allele.post_mapped, + "mapping_api_version": mapping_record.mapping_api_version, }, "entId": variant.urn, # type: ignore "entIri": f"{MAVEDB_FRONTEND_URL}/score-sets/{quote_plus(variant.score_set.urn)}?variant={quote_plus(variant.urn)}", # type: ignore } ] } - if mapped_variant is not None: - entity["MaveDBMapping"][0]["entContent"]["pre_mapped"] = mapped_variant.pre_mapped - entity["MaveDBMapping"][0]["entContent"]["post_mapped"] = mapped_variant.post_mapped - entity["MaveDBMapping"][0]["entContent"]["mapping_api_version"] = mapped_variant.mapping_api_version - return entity def construct_ldh_submission( - variant_content: list[tuple[str, Variant, Optional[MappedVariant]]], + variant_content: list[tuple[str, Variant, MappingRecord, Allele]], ) -> list[LdhSubmission]: content_submission: list[LdhSubmission] = [] - for hgvs, variant, mapped_variant in variant_content: + for hgvs, variant, mapping_record, allele in variant_content: subject = construct_ldh_submission_subject(hgvs) event = construct_ldh_submission_event(subject) - entity = construct_ldh_submission_entity(variant, mapped_variant) + entity = construct_ldh_submission_entity(variant, mapping_record, allele) content_submission.append( { diff --git a/src/mavedb/models/score_set.py b/src/mavedb/models/score_set.py index ccdcb73d5..b61704ffe 100644 --- a/src/mavedb/models/score_set.py +++ b/src/mavedb/models/score_set.py @@ -74,7 +74,7 @@ class ScoreSet(Base): __tablename__ = "scoresets" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) # TODO(#372) urn = Column(String(64), default=generate_temp_urn, index=True, nullable=True, unique=True) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 5a65cc728..27aa1b962 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -3,7 +3,6 @@ This module contains jobs for submitting mapped variants to ClinGen services: - ClinGen Allele Registry (CAR) for allele registration - ClinGen Linked Data Hub (LDH) for data submission -- Variant linking and association management These jobs enable integration with the ClinGen ecosystem for clinical variant interpretation and data sharing. @@ -12,10 +11,12 @@ import asyncio import functools import logging +from dataclasses import dataclass, field from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, @@ -29,9 +30,11 @@ ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params @@ -41,6 +44,50 @@ logger = logging.getLogger(__name__) +@dataclass +class _AlleleEntry: + post_mapped: dict | None + existing_caid: str | None + # Variants for which THIS allele is the authoritative measurement — the only ones that receive a + # per-variant VAS row. INTERIM BANDAID (do not deploy as final): keying clingen's per-variant + # status to the single authoritative link sidesteps the multiple "current" rows a full allele + # fan-out would write for one variant. Durable fix is an allele-level event log; rationale and + # migration seam in docs/design/allele-annotation-status.md. + authoritative_variant_ids: list[int] = field(default_factory=list) + + +def _annotate_caid( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a CLINGEN_ALLELE_ID annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for clingen's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (who caused the registration). See + docs/design/allele-annotation-status.md. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + version=None, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + @with_pipeline_management async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ @@ -58,11 +105,11 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager (JobManager): Manager for job lifecycle and DB operations Side Effects: - - Updates MappedVariant records with ClinGen Allele IDs + - Updates Allele records with ClinGen Allele IDs - Submits data to ClinGen Allele Registry Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele counts (submitted/registered/already-registered/failed). """ # Get the job definition we are working on job = job_manager.get_job() @@ -70,11 +117,10 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: _job_required_params = ["score_set_id", "correlation_id"] validate_job_params(_job_required_params, job) - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - # Setup initial context and progress job_manager.save_to_context( { "application": "mavedb-worker", @@ -86,7 +132,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) - # Ensure we've enabled ClinGen submission if not CLIN_GEN_SUBMISSION_ENABLED: logger.warning( msg="ClinGen submission is disabled via configuration, skipping submission of mapped variants to CAR.", @@ -95,7 +140,6 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.skipped(data={"reason": "ClinGen submission disabled"}) - # Check for CAR submission endpoint if not CAR_SUBMISSION_ENDPOINT: logger.warning( msg="ClinGen Allele Registry submission is disabled (no submission endpoint), unable to complete submission of mapped variants to CAR.", @@ -107,183 +151,300 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: failure_category=FailureCategory.CONFIGURATION_ERROR, ) - # Fetch mapped variants with post-mapped data for the score set - variant_post_mapped_objects = job_manager.db.execute( - select(MappedVariant.id, MappedVariant.post_mapped) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) - ).all() - - # Track total variants to submit - job_manager.save_to_context({"total_variants_to_submit_car": len(variant_post_mapped_objects)}) - if not variant_post_mapped_objects: + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + job_manager.save_to_context({"total_allele_variant_pairs": len(allele_rows)}) + if not allele_rows: logger.warning( - msg="No current mapped variants with post mapped metadata were found for this score set. Skipping CAR submission.", + msg="No current alleles found for this score set. Skipping CAR submission.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"submitted_count": 0, "matched_count": 0}) + return JobExecutionOutcome.succeeded( + data={ + "submitted_allele_count": 0, + "registered_allele_count": 0, + "already_registered_allele_count": 0, + "failed_allele_count": 0, + } + ) - job_manager.update_progress( - 10, 100, f"Preparing {len(variant_post_mapped_objects)} mapped variants for CAR submission." - ) + # Group by allele_id: one allele may serve multiple variants (cross-score-set dedup). + allele_data: dict[int, _AlleleEntry] = {} + for row in allele_rows: + if row.allele_id not in allele_data: + allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) + + if row.is_authoritative: + allele_data[row.allele_id].authoritative_variant_ids.append(row.variant_id) - # Build HGVS strings for submission. Don't do duplicate submissions-- store mapped variant IDs by HGVS. - # Variants that can't produce an HGVS string are annotated as failures immediately. annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - variant_post_mapped_hgvs: dict[str, list[int]] = {} - no_hgvs_count = 0 - for mapped_variant_id, post_mapped in variant_post_mapped_objects: - # Intentionally not combine_cis=True: multi-variant cis-phased blocks have no single - # CAID, so they are skipped here pending ClinGen guidance on how to register them - # (https://github.com/VariantEffect/mavedb-api/issues/764). - hgvs_for_post_mapped = get_hgvs_from_post_mapped(post_mapped) - if not hgvs_for_post_mapped: - no_hgvs_count += 1 + # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID + # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele + # units — and they cover every allele submitted, including the RT-derived ones that produce no + # per-variant status row. Each allele has exactly one outcome (submitted once → one response), so + # these sets are disjoint by construction. (Per-variant VAS rows are still written via the + # authoritative link below — that is the interim bandaid, separate from these operation counts.) + linked_allele_ids: set[int] = set() + preexisting_allele_ids: set[int] = set() + failed_allele_ids: set[int] = set() + + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. + preexisting = [aid for aid, entry in allele_data.items() if entry.existing_caid] if not force_reregister else [] + for allele_id in preexisting: + entry = allele_data[allele_id] + + preexisting_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "preexisting"}, + ) + + # Alleles that need CAR submission: new ones, or all when force_reregister=True. + pending_allele_ids = [aid for aid, entry in allele_data.items() if force_reregister or not entry.existing_caid] + job_manager.update_progress(10, 100, f"Preparing {len(pending_allele_ids)} alleles for CAR submission.") + + # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS + # (combine_cis defaults to False); those alleles are annotated as failures immediately. + hgvs_to_allele_ids: dict[str, list[int]] = {} + for allele_id in pending_allele_ids: + entry = allele_data[allele_id] + hgvs = get_hgvs_from_post_mapped(entry.post_mapped) + + if hgvs: + hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) + + # Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as + # preexisting rather than failing the variant. + elif entry.existing_caid: + preexisting_allele_ids.add(allele_id) logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant_id}. Skipping submission of this variant.", + msg=( + f"Could not construct HGVS for allele {allele_id} during force re-registration " + f"(existing CAID: {entry.existing_caid!r}). Reconfirmation skipped; existing CAID retained." + ), extra=job_manager.logging_context(), ) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "reconfirmation_skipped"}, + ) - mapped_variant = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id == mapped_variant_id) - ).one() - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + # No HGVS-- un-submittable. + else: + failed_allele_ids.add(allele_id) + logger.warning( + msg=f"Could not construct HGVS for allele {allele_id}. Skipping CAR submission.", + extra=job_manager.logging_context(), + ) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Could not extract a valid HGVS string from post-mapped variant data.", - "annotation_metadata": {}, - }, - current=True, + error_message="Could not extract a valid HGVS string from post-mapped allele data.", ) - continue - if hgvs_for_post_mapped in variant_post_mapped_hgvs: - variant_post_mapped_hgvs[hgvs_for_post_mapped].append(mapped_variant_id) - else: - variant_post_mapped_hgvs[hgvs_for_post_mapped] = [mapped_variant_id] + job_manager.save_to_context({"unique_hgvs_to_submit_car": len(hgvs_to_allele_ids)}) - job_manager.save_to_context({"unique_variants_to_submit_car": len(variant_post_mapped_hgvs)}) - job_manager.update_progress(15, 100, "Submitting mapped variants to CAR.") + # Distinct alleles actually sent to CAR this run (pending alleles that yielded HGVS). + submitted_allele_ids: set[int] = { + allele_id for allele_ids in hgvs_to_allele_ids.values() for allele_id in allele_ids + } + + def _outcome_data() -> dict[str, int]: + return { + "submitted_allele_count": len(submitted_allele_ids), + "registered_allele_count": len(linked_allele_ids), + "already_registered_allele_count": len(preexisting_allele_ids), + "failed_allele_count": len(failed_allele_ids), + } - # Do submission + # All pending alleles failed HGVS extraction; annotations already written above. + if not hgvs_to_allele_ids: + annotation_manager.flush() + job_manager.db.flush() + if preexisting_allele_ids: + return JobExecutionOutcome.succeeded(data=_outcome_data()) + + return JobExecutionOutcome.failed( + reason=f"No submittable alleles for score set {score_set.urn}.", + data=_outcome_data(), + failure_category=FailureCategory.DEPENDENCY_FAILURE, + ) + + job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(variant_post_mapped_hgvs.keys()) + hgvs_list = list(hgvs_to_allele_ids.keys()) registered_alleles = car_service.dispatch_submissions(hgvs_list) job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") - # CAR returns one response per submitted HGVS in the same order (see CAR API docs). - # Zip the submissions with the responses and annotate each based on success or error. - linked_count = 0 - error_count = 0 + # Bulk-load every allele that could be linked in one query, keyed by id, rather than + # issuing a SELECT per CAID inside the response loop. + alleles_by_id = { + allele.id: allele + for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() + } + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail the whole batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing the entire batch." + ), + extra=job_manager.logging_context(), + ) - for hgvs_string, response in zip(hgvs_list, registered_alleles): - mapped_variant_ids = variant_post_mapped_hgvs[hgvs_string] - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(mapped_variant_ids)) - ).all() + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] if "errorType" in response: - error_count += 1 logger.warning( msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ + "submitted_hgvs": hgvs_string, + "car_error_type": response.get("errorType"), + "car_error_message": response.get("message"), + }, + ) + + continue + + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: + logger.error( + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", + extra=job_manager.logging_context(), + ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, + failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) + + continue - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + caid = caid_iri.split("/")[-1] + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # CAID is immutable — a different value returned by CAR is a hard invariant + # violation. Do not overwrite; record a failure with full audit context. + if prior_caid and prior_caid != caid: + logger.error( + msg=( + f"CAR returned a different CAID for allele {allele_id}: " + f"stored={prior_caid!r}, returned={caid!r}. " + "Not overwriting. Investigate immediately." + ), + extra=job_manager.logging_context(), + ) + + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": { - "submitted_hgvs": hgvs_string, - "car_error_type": response.get("errorType"), - "car_error_message": response.get("message"), - }, + error_message="CAR returned a CAID that conflicts with the stored value.", + metadata={ + "clingen_allele_id": prior_caid, + "conflicting_caid": caid, + "submitted_hgvs": hgvs_string, }, - current=True, ) - else: - linked_count += 1 - caid = response["@id"].split("/")[-1] - for mapped_variant in mapped_variants: - mapped_variant.clingen_allele_id = caid - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"clingen_allele_id": caid}}, - current=True, + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + registration_source = "reconfirmed" if prior_caid else "this_run" + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "registration_source": registration_source}, ) - # Any HGVS strings CAR did not respond to (network drop, service-side omission). - # Use EXTERNAL_SERVICE_REJECTED for explicit CAR errors, EXTERNAL_API_ERROR for silent failures. - no_response_hgvs = hgvs_list[len(registered_alleles) :] - for hgvs_string in no_response_hgvs: - mapped_variants = job_manager.db.scalars( - select(MappedVariant).where(MappedVariant.id.in_(variant_post_mapped_hgvs[hgvs_string])) - ).all() - for mapped_variant in mapped_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=AnnotationStatus.FAILED, + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_data[allele_id].authoritative_variant_ids, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to register variant with ClinGen Allele Registry.", - "annotation_metadata": {"submitted_hgvs": hgvs_string}, - }, - current=True, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, ) annotation_manager.flush() - failed_count = no_hgvs_count + error_count + len(no_response_hgvs) + outcome_data = _outcome_data() - # When all registrations fail we will not be able to render any annotations. Fail the job - # to explicitly halt the pipeline. - if linked_count == 0: - error_message = f"CAR submission failed for all {len(hgvs_list)} variants in score set {score_set.urn}." + # When no allele ended up with a CAID (none linked this run, none already registered), the + # pipeline cannot continue — downstream jobs need CAIDs to function. + if not linked_allele_ids and not preexisting_allele_ids: + error_message = ( + f"CAR submission failed for all {outcome_data['submitted_allele_count']} " + f"submitted alleles in score set {score_set.urn}." + ) logger.error(msg=error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": len(hgvs_list), "matched_count": 0, "failed_count": failed_count}, + data=outcome_data, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) - if failed_count > 0: - # CAR rejections are typically per-variant data quality issues (e.g. invalid HGVS) rather than - # systemic failures. Per-variant AnnotationStatus.FAILED records are already written above for - # traceability. We continue the pipeline so that successfully registered variants still receive - # downstream annotations (warm_clingen_cache, gnomAD, ClinVar, HGVS, translations). + if outcome_data["failed_allele_count"] > 0: logger.warning( - msg=f"CAR submission failed for {failed_count} of {len(hgvs_list)} variants in score set {score_set.urn}.", + msg=f"CAR submission failed for {outcome_data['failed_allele_count']} alleles in score set {score_set.urn}.", extra=job_manager.logging_context(), ) logger.info(msg="Completed CAR mapped resource submission", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(hgvs_list), "matched_count": linked_count, "failed_count": failed_count} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) @with_pipeline_management @@ -306,7 +467,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: - Submits data to ClinGen Linked Data Hub Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-variant submitted/failed counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -334,14 +495,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ldh_service = ClinGenLdhService(url=LDH_SUBMISSION_ENDPOINT) ldh_service.authenticate() - # Fetch mapped variants with post-mapped data for the score set + # Fetch each variant's authoritative allele for the score set. Post-mapped data and HGVS + # come from the Allele; pre-mapped data and the mapping API version come from the + # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each + # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, + # which registers a CAID per allele). variant_objects = job_manager.db.execute( - select(Variant, MappedVariant) - .join(MappedVariant) - .join(ScoreSet) - .where(ScoreSet.urn == score_set.urn) - .where(MappedVariant.post_mapped.is_not(None)) - .where(MappedVariant.current.is_(True)) + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) ).all() # Track total variants to submit @@ -359,19 +527,19 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # Build submission content variant_content = [] variant_for_urn = {} - for variant, mapped_variant in variant_objects: + for variant, mapping_record, allele in variant_objects: # See the note above: cis-phased blocks are skipped here pending ClinGen guidance # (https://github.com/VariantEffect/mavedb-api/issues/764). - variation = get_hgvs_from_post_mapped(mapped_variant.post_mapped) + variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: logger.warning( - msg=f"Could not construct a valid HGVS string for mapped variant {mapped_variant.id}. Skipping submission of this variant.", + msg=f"Could not construct a valid HGVS string for allele {allele.id} (variant {variant.urn}). Skipping submission of this variant.", extra=job_manager.logging_context(), ) continue - variant_content.append((variation, variant, mapped_variant)) + variant_content.append((variation, variant, mapping_record, allele)) variant_for_urn[variant.urn] = variant if not variant_content: @@ -409,7 +577,15 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) submitted_urn = success["data"]["entId"] - submitted_variant = variant_for_urn[submitted_urn] + submitted_variant = variant_for_urn.get(submitted_urn) + if submitted_variant is None: + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. + logger.warning( + msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", + extra=job_manager.logging_context(), + ) + continue annotation_manager.add_annotation( variant_id=submitted_variant.id, @@ -427,7 +603,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # especially when submission occurred in batch. Save all failures generically here. # Note that failures may not be present in the submission failures list, but they are # guaranteed to be absent from the successes list. - for failure_urn in set(variant_for_urn.keys()) - submitted_variant_urns: + failed_variant_urns = set(variant_for_urn.keys()) - submitted_variant_urns + for failure_urn in failed_variant_urns: logger.error( msg=f"Failed to submit mapped variant to LDH: {failure_urn}", extra=job_manager.logging_context(), @@ -449,9 +626,15 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: annotation_manager.flush() + # Report per-variant counts (matching the annotations written above), not the per-batch + # counts returned by the service — the two use different denominators. + submitted_count = len(submitted_variant_urns) + failed_count = len(failed_variant_urns) + if submission_failures: logger.warning( - msg=f"LDH mapped resource submission encountered {len(submission_failures)} failures.", + msg=f"LDH mapped resource submission encountered {len(submission_failures)} batch failures " + f"({failed_count} variants unconfirmed).", extra=job_manager.logging_context(), ) @@ -465,7 +648,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.db.flush() return JobExecutionOutcome.failed( reason=error_message, - data={"submitted_count": 0, "failed_count": len(submission_failures)}, + data={"submitted_count": submitted_count, "failed_count": failed_count}, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -475,6 +658,4 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"submitted_count": len(submission_successes), "failed_count": len(submission_failures)} - ) + return JobExecutionOutcome.succeeded(data={"submitted_count": submitted_count, "failed_count": failed_count}) diff --git a/src/mavedb/worker/jobs/external_services/clingen_cache.py b/src/mavedb/worker/jobs/external_services/clingen_cache.py index 10890f23f..d530d881f 100644 --- a/src/mavedb/worker/jobs/external_services/clingen_cache.py +++ b/src/mavedb/worker/jobs/external_services/clingen_cache.py @@ -15,11 +15,10 @@ from sqlalchemy import select from mavedb.lib.clingen.allele_registry import get_clingen_allele_data +from mavedb.lib.clingen.alleles import get_alleles_for_score_set from mavedb.lib.clingen.constants import CLINGEN_CACHE_WARMING_CONCURRENCY from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -55,19 +54,16 @@ async def warm_clingen_cache(ctx: dict, job_id: int, job_manager: JobManager) -> job_manager.update_progress(0, 100, "Starting ClinGen cache pre-warming.") logger.info("Starting ClinGen cache pre-warming", extra=job_manager.logging_context()) - # Get distinct clingen_allele_ids for this score set's current mapped variants - allele_ids = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.isnot(None), - # Exclude multi-variant IDs (comma-separated) — they can't be fetched individually - MappedVariant.clingen_allele_id.not_like("%,%"), - ) - .distinct() - ).all() + # Get distinct clingen_allele_ids registered for alleles in this score set. + # Exclude None and comma-separated multi-variant IDs that can't be fetched individually. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_ids = list( + { + row.clingen_allele_id + for row in allele_rows + if row.clingen_allele_id is not None and "," not in row.clingen_allele_id + } + ) total = len(allele_ids) job_manager.save_to_context({"total_allele_ids_to_warm": total}) diff --git a/tests/lib/clingen/test_content_constructors.py b/tests/lib/clingen/test_content_constructors.py index 7aab47f76..f691a4497 100644 --- a/tests/lib/clingen/test_content_constructors.py +++ b/tests/lib/clingen/test_content_constructors.py @@ -11,7 +11,6 @@ ) from mavedb.lib.clingen.constants import LDH_ENTITY_NAME, LDH_SUBMISSION_TYPE from mavedb import __version__ -import pytest from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -54,10 +53,8 @@ def test_construct_ldh_submission_event(): } -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None - result = construct_ldh_submission_entity(mock_variant, mapped_variant) +def test_construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele): + result = construct_ldh_submission_entity(mock_variant, mock_mapping_record, mock_allele) assert "MaveDBMapping" in result assert len(result["MaveDBMapping"]) == 1 @@ -65,15 +62,9 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ assert mapping["entContent"]["mavedb_id"] == VALID_VARIANT_URN assert mapping["entContent"]["score"] == 1.0 - - if has_mapped_variant: - assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X - assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" - else: - assert "pre_mapped" not in mapping["entContent"] - assert "post_mapped" not in mapping["entContent"] - assert "mapping_api_version" not in mapping["entContent"] + assert mapping["entContent"]["pre_mapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["post_mapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert mapping["entContent"]["mapping_api_version"] == "pytest.mapping.1.0" assert mapping["entId"] == VALID_VARIANT_URN assert ( @@ -82,12 +73,10 @@ def test_construct_ldh_submission_entity(mock_variant, mock_mapped_variant, has_ ) -@pytest.mark.parametrize("has_mapped_variant", [(True), (False)]) -def test_construct_ldh_submission(mock_variant, mock_mapped_variant, has_mapped_variant: bool): - mapped_variant = mock_mapped_variant if has_mapped_variant else None +def test_construct_ldh_submission(mock_variant, mock_mapping_record, mock_allele): variant_content = [ - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), - (TEST_HGVS_IDENTIFIER, mock_variant, mapped_variant), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), + (TEST_HGVS_IDENTIFIER, mock_variant, mock_mapping_record, mock_allele), ] uuid_1 = UUID("12345678-1234-5678-1234-567812345678") diff --git a/tests/lib/conftest.py b/tests/lib/conftest.py index 2ac2183fb..adfe2a1b8 100644 --- a/tests/lib/conftest.py +++ b/tests/lib/conftest.py @@ -307,6 +307,23 @@ def mock_mapped_variant(mock_variant): return mv +@pytest.fixture +def mock_mapping_record(mock_mapped_variant): + # Pre-mapped data and the mapping API version now live on the per-variant MappingRecord. + rec = mock.Mock() + rec.pre_mapped = mock_mapped_variant.pre_mapped + rec.mapping_api_version = mock_mapped_variant.mapping_api_version + return rec + + +@pytest.fixture +def mock_allele(mock_mapped_variant): + # Post-mapped data now lives on the (cross-variant deduped) Allele. + allele = mock.Mock() + allele.post_mapped = mock_mapped_variant.post_mapped + return allele + + @pytest.fixture def mock_mapped_variant_with_functional_calibration_score_set( mock_mapped_variant, mock_variant_with_functional_calibration_score_set diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 616263dfa..76b1b0b2d 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -5,14 +5,16 @@ pytest.importorskip("arq") from asyncio.unix_events import _UnixSelectorEventLoop +from copy import deepcopy from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.clingen import ( @@ -51,9 +53,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_mappings( self, @@ -76,9 +78,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( self, @@ -101,9 +103,9 @@ async def test_submit_score_set_mappings_to_car_submission_endpoint_not_set( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 async def test_submit_score_set_mappings_to_car_no_registered_alleles( self, @@ -147,11 +149,11 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -184,18 +186,18 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( dummy_variant_mapping_job_run, ) - # Build error responses for every submitted HGVS — CAR explicitly rejected all of them - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele (same VRS digest). Build an error response for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "errorType": "InvalidHGVS", - "hgvs": get_hgvs_from_post_mapped(mv.post_mapped) or "", + "hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", "message": "Invalid HGVS expression.", "description": "", - "inputLine": get_hgvs_from_post_mapped(mv.post_mapped) or "", - "position": str(i), + "inputLine": get_hgvs_from_post_mapped(alleles[0].post_mapped) or "", + "position": "0", } - for i, mv in enumerate(mapped_variants) ] with ( @@ -215,11 +217,11 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # 1 allele failed → all 4 variant annotations are failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -228,6 +230,211 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( assert ann.status == "failed" assert ann.annotation_type == "clingen_allele_id" + async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_annotation( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A variant linked to an authoritative AND a derived allele gets exactly one VAS row (from the + authoritative link), while the derived allele is still registered with a CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # The sample's 4 variants dedup to one authoritative allele. + authoritative_allele = session.scalars(select(Allele)).one() + + # Attach a second, derived (is_authoritative=False) allele to one variant's current mapping + # record. It shares the authoritative allele's HGVS (different vrs_digest) so it is submitted + # under the same line — the realistic shape is a different level, but identical HGVS is enough + # to exercise the multi-allele-per-variant fan-out. + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + derived_allele = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-derived", + level=authoritative_allele.level, + post_mapped={**deepcopy(authoritative_allele.post_mapped), "id": "derived-allele-id"}, + ) + session.add(derived_allele) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived_allele.id, + is_authoritative=False, + ) + ) + session.flush() + + def fake_dispatch(hgvs_list): + return [{"@id": f"CA{idx}", "type": "nucleotide", "genomicAlleles": []} for idx, _ in enumerate(hgvs_list)] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # Exactly one current VAS row per variant (4 total) — no duplicate from the derived allele. + current_statuses = session.scalars( + select(VariantAnnotationStatus).where( + VariantAnnotationStatus.annotation_type == "clingen_allele_id", + VariantAnnotationStatus.current.is_(True), + ) + ).all() + assert len(current_statuses) == 4 + assert len({s.variant_id for s in current_statuses}) == 4 + + # Both alleles registered: registration breadth is preserved even though the derived allele + # produced no VAS row. + session.refresh(authoritative_allele) + session.refresh(derived_allele) + assert authoritative_allele.clingen_allele_id is not None + assert derived_allele.clingen_allele_id is not None + + async def test_submit_score_set_mappings_to_car_response_count_mismatch( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # All 4 variants share 1 allele → 1 submitted HGVS, but CAR returns 2 results. The count + # violates the one-result-per-input contract, so positional alignment can't be trusted + # and the whole batch must be rejected without writing any CAID. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ + {"@id": "CA111111", "type": "nucleotide", "genomicAlleles": []}, + {"@id": "CA222222", "type": "nucleotide", "genomicAlleles": []}, + ] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID written — neither of the returned values is trusted. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.failure_category == "external_api_error" + + async def test_submit_score_set_mappings_to_car_malformed_response( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + # Create mappings in the score set + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # CAR returns a response that is neither an error (no "errorType") nor a registration + # (no "@id"). This must be surfaced as a rejection, not crash the loop with a KeyError. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [{"type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.FAILED + + # No CAID assigned, and all 4 variant annotations are failed as rejected. + assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.failure_category == "external_service_rejected" + async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, mock_worker_ctx, @@ -252,13 +459,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles with repeated HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() + # 4 variants share 1 allele; CAR returns 1 response for the 1 submitted HGVS + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 registered_alleles_mock = [ { "@id": "CA_DUPLICATE", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mapped_variants[0].post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -267,11 +475,6 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", return_value=registered_alleles_mock, ), - # Patch get_hgvs_from_post_mapped to return the same HGVS for all variants - patch( - "mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", - return_value=get_hgvs_from_post_mapped(mapped_variants[0].post_mapped), - ), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), ): @@ -284,13 +487,12 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == "CA_DUPLICATE" + # 1 allele received the CAID + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 1 + assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -312,7 +514,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures (some matched, some not) result in a succeeded outcome with failure annotations.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -324,16 +526,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Get mapped variants; return a CAR response that only matches the first variant - mapped_variants = session.scalars(select(MappedVariant)).all() - assert len(mapped_variants) == 4 + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -353,31 +553,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 - # Verify only the first variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses: 1 success, 3 failed + # All 4 variant annotations succeeded success_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.status == "success", ) ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) - ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(success_annotations) == 4 async def test_submit_score_set_mappings_to_car_hgvs_not_found( self, @@ -403,32 +594,11 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant) - .join(Variant) - .where(Variant.score_set_id == submit_score_set_mappings_to_car_sample_job_run.job_params["score_set_id"]) - ).all() - - # Patch ClinGenAlleleRegistryService to return registered alleles - registered_alleles_mock = [ - { - "@id": f"CA{mv.id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], - } - for mv in mapped_variants - ] - + # Patch get_hgvs_from_post_mapped to return None for all alleles with ( - patch( - "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", - return_value=registered_alleles_mock, - ), - # Patch get_hgvs_from_post_mapped to not find any HGVS in registered alleles - patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + patch("mavedb.worker.jobs.external_services.clingen.get_hgvs_from_post_mapped", return_value=None), ): result = await submit_score_set_mappings_to_car( mock_worker_ctx, @@ -439,11 +609,11 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -519,20 +689,16 @@ async def test_submit_score_set_mappings_to_car_success( dummy_variant_mapping_job_run, ) - # Get the mapped variants from score set before submission - mapped_variants = session.scalars( - select(MappedVariant).join(Variant).where(Variant.score_set_id == sample_score_set.id) - ).all() - assert len(mapped_variants) == 4 + # All 4 variants share 1 allele (same VRS digest from construct_mock_mapping_output) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 - # Patch ClinGenAlleleRegistryService to return registered alleles registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -552,13 +718,12 @@ async def test_submit_score_set_mappings_to_car_success( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 4 - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -567,6 +732,192 @@ async def test_submit_score_set_mappings_to_car_success( assert ann.status == "success" assert ann.annotation_type == "clingen_allele_id" + async def test_submit_score_set_mappings_to_car_preexisting( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Already-registered allele is re-annotated as preexisting, not re-submitted.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + # Pre-set the CAID on the allele to simulate prior registration + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_PRIOR" + session.flush() + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + ) as mock_dispatch, + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # CAR should NOT be called since the allele is already registered + mock_dispatch.assert_not_called() + assert result.status == JobStatus.SUCCEEDED + assert result.data["already_registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 0 + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "success" + assert ann.annotation_metadata["registration_source"] == "preexisting" + assert ann.annotation_metadata["clingen_allele_id"] == "CA_PRIOR" + + async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration that returns the same CAID is a success with registration_source=reconfirmed.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_CONFIRMED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + registered_alleles_mock = [{"@id": "CA_CONFIRMED", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["registered_allele_count"] == 1 + assert result.data["submitted_allele_count"] == 1 + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "success" + assert ann.annotation_metadata["registration_source"] == "reconfirmed" + + async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """Force re-registration returning a different CAID fails without overwriting the stored CAID.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + + allele = session.scalars(select(Allele)).first() + allele.clingen_allele_id = "CA_STORED" + session.flush() + + submit_score_set_mappings_to_car_sample_job_run.job_params = { + **submit_score_set_mappings_to_car_sample_job_run.job_params, + "force_reregister": True, + } + session.flush() + + # CAR returns a DIFFERENT CAID — this is an invariant violation + registered_alleles_mock = [{"@id": "CA_DIFFERENT", "type": "nucleotide", "genomicAlleles": []}] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + return_value=registered_alleles_mock, + ), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # Job fails because all CAID-returning submissions had a conflict + assert result.status == JobStatus.FAILED + + # CAID must NOT have been overwritten + session.refresh(allele) + assert allele.clingen_allele_id == "CA_STORED" + + annotation_statuses = session.scalars( + select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + ).all() + assert len(annotation_statuses) == 4 + for ann in annotation_statuses: + assert ann.status == "failed" + assert ann.annotation_metadata["clingen_allele_id"] == "CA_STORED" + assert ann.annotation_metadata["conflicting_caid"] == "CA_DIFFERENT" + @pytest.mark.integration @pytest.mark.asyncio @@ -597,15 +948,14 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -623,17 +973,16 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) + assert len(annotation_statuses) == 4 for ann in annotation_statuses: assert ann.status == "success" @@ -666,15 +1015,14 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -692,17 +1040,16 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == len(mapped_variants) + assert len(annotation_statuses) == 4 for ann in annotation_statuses: assert ann.status == "success" @@ -738,9 +1085,9 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SKIPPED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -777,9 +1124,9 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -808,9 +1155,9 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() @@ -862,11 +1209,11 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -903,9 +1250,6 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( # Patch ClinGenAlleleRegistryService to return only errors with no linked alleles registered_alleles_mock = [ {"errorType": "InvalidHGVS", "hgvs": "test"}, - {"errorType": "InvalidHGVS", "hgvs": "test2"}, - {"errorType": "InvalidHGVS", "hgvs": "test3"}, - {"errorType": "InvalidHGVS", "hgvs": "test4"}, ] with ( @@ -925,11 +1269,11 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed + # Verify annotation statuses were rendered as failed — 4 variants, all failed annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -952,7 +1296,7 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """Test that partial CAR failures result in SUCCEEDED status with per-variant failure annotations committed.""" + """All 4 variants share 1 allele; CAR returns 1 success → 1 registered allele, 0 failed, job SUCCEEDED.""" # Create mappings in the score set await create_mappings_in_score_set( session, @@ -964,14 +1308,14 @@ async def test_submit_score_set_mappings_to_car_partial_failure( dummy_variant_mapping_job_run, ) - # Return a CAR response that only matches the first variant's HGVS - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + registered_alleles_mock = [ { - "@id": f"CA{mapped_variants[0].id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } ] @@ -991,31 +1335,22 @@ async def test_submit_score_set_mappings_to_car_partial_failure( mock_send_slack_error.assert_not_called() assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["matched_count"] == 1 - assert result.data["failed_count"] == 3 + assert result.data["registered_allele_count"] == 1 + assert result.data["failed_allele_count"] == 0 - # Verify the successfully matched variant got a CAID - variants_with_caid = session.scalars( - select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None)) - ).all() - assert len(variants_with_caid) == 1 - assert variants_with_caid[0].clingen_allele_id == f"CA{mapped_variants[0].id}" + # 1 allele got a CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses: 1 success, 3 failed + # All 4 variant annotations succeeded success_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.status == "success", ) ).all() - failed_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "failed", - ) - ).all() - assert len(success_annotations) == 1 - assert len(failed_annotations) == 3 + assert len(success_annotations) == 4 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,22 +1381,17 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno dummy_variant_mapping_job_run, ) - # Return a CAR response where: first variant succeeds, second has explicit CAR error, rest are silent failures - mapped_variants = session.scalars(select(MappedVariant)).all() - first_hgvs = get_hgvs_from_post_mapped(mapped_variants[0].post_mapped) - second_hgvs = get_hgvs_from_post_mapped(mapped_variants[1].post_mapped) + # All 4 variants share 1 allele. CAR returns an explicit error for that 1 HGVS. + alleles = session.scalars(select(Allele)).all() + assert len(alleles) == 1 + allele_hgvs = get_hgvs_from_post_mapped(alleles[0].post_mapped) registered_alleles_mock = [ - { - "@id": f"CA{mapped_variants[0].id}", - "type": "nucleotide", - "genomicAlleles": [{"hgvs": first_hgvs}], - }, { "errorType": "InvalidHGVS", - "hgvs": second_hgvs, + "hgvs": allele_hgvs, "message": "The HGVS string is invalid.", "description": "error", - "inputLine": second_hgvs, + "inputLine": allele_hgvs, "position": "0", }, ] @@ -1079,31 +1409,18 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # Verify the variant whose HGVS returned an explicit CAR error has error details in annotation_metadata. - # Only 1 annotation should have EXTERNAL_SERVICE_REJECTED since only one CAR error was in the response. + # All 4 variant annotations should have EXTERNAL_SERVICE_REJECTED since the 1 shared allele was rejected car_rejected_annotations = session.scalars( select(VariantAnnotationStatus).where( VariantAnnotationStatus.annotation_type == "clingen_allele_id", VariantAnnotationStatus.failure_category == "external_service_rejected", ) ).all() - assert len(car_rejected_annotations) == 1 - rejected = car_rejected_annotations[0] - assert rejected.annotation_metadata["submitted_hgvs"] == second_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." - - # The remaining 2 failures (variants 3 and 4) got no CAR response — silent failures get EXTERNAL_API_ERROR. - silent_failure_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_api_error", - ) - ).all() - assert len(silent_failure_annotations) == 2 - for ann in silent_failure_annotations: - assert ann.annotation_metadata["submitted_hgvs"] is not None - assert "car_error_type" not in ann.annotation_metadata + assert len(car_rejected_annotations) == 4 + for rejected in car_rejected_annotations: + assert rejected.annotation_metadata["submitted_hgvs"] == allele_hgvs + assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" + assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1185,15 +1502,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1214,13 +1530,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( session.refresh(submit_score_set_mappings_to_car_sample_job_run) assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -1255,15 +1570,14 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( dummy_variant_mapping_job_run, ) - # Patch ClinGenAlleleRegistryService to return registered alleles - mapped_variants = session.scalars(select(MappedVariant)).all() + # All 4 variants share 1 allele; build 1 CAR response + alleles = session.scalars(select(Allele)).all() registered_alleles_mock = [ { - "@id": f"CA{mv.id}", + "@id": f"CA{alleles[0].id}", "type": "nucleotide", - "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(mv.post_mapped)}], + "genomicAlleles": [{"hgvs": get_hgvs_from_post_mapped(alleles[0].post_mapped)}], } - for mv in mapped_variants ] with ( @@ -1288,13 +1602,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.SUCCEEDED - # Verify variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == len(mapped_variants) - for variant in variants: - assert variant.clingen_allele_id == f"CA{variant.id}" + # 1 allele received the CAID + alleles_with_caid = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles_with_caid) == 1 + assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" - # Verify annotation statuses were rendered as success + # 4 per-variant annotations — all success annotation_statuses = session.scalars( select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") ).all() @@ -1350,9 +1663,9 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert submit_score_set_mappings_to_car_sample_job_run.status == JobStatus.ERRORED assert submit_score_set_mappings_to_car_sample_job_run.error_message == "ClinGen service error" - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars( @@ -1413,9 +1726,9 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl session.refresh(submit_score_set_mappings_to_car_sample_pipeline) assert submit_score_set_mappings_to_car_sample_pipeline.status == PipelineStatus.FAILED - # Verify no variants have CAIDs assigned - variants = session.scalars(select(MappedVariant).where(MappedVariant.clingen_allele_id.isnot(None))).all() - assert len(variants) == 0 + # Verify no alleles have CAIDs assigned + alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(alleles) == 0 # Verify no annotation statuses were created annotation_statuses = session.scalars( diff --git a/tests/worker/jobs/external_services/test_clingen_cache.py b/tests/worker/jobs/external_services/test_clingen_cache.py index a55eb6b88..6d95b917a 100644 --- a/tests/worker/jobs/external_services/test_clingen_cache.py +++ b/tests/worker/jobs/external_services/test_clingen_cache.py @@ -4,11 +4,14 @@ pytest.importorskip("arq") +from datetime import date from unittest.mock import AsyncMock, patch from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import JobStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.worker.jobs.external_services.clingen_cache import warm_clingen_cache @@ -17,6 +20,51 @@ pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +def _make_allele_with_caid(session, score_set_id: int, urn_suffix: str, caid: str | None, vrs_digest: str) -> Allele: + """Create a Variant → MappingRecord → Allele chain and return the Allele. + + get_alleles_for_score_set joins: Allele ← MappingRecordAllele (current) ← MappingRecord (current) ← Variant. + Leaving valid_to=NULL on MappingRecord and MappingRecordAllele makes them current. + """ + variant = Variant( + urn=f"urn:variant:{urn_suffix}", + score_set_id=score_set_id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.flush() + + allele = Allele( + vrs_digest=vrs_digest, + level="genomic", + post_mapped={"type": "Allele", "location": {"sequenceReference": {"refgetAccession": vrs_digest}}}, + clingen_allele_id=caid, + ) + session.add(allele) + session.flush() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(mapping_record) + session.flush() + + link = MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + session.add(link) + session.flush() + + return allele + + @pytest.mark.unit @pytest.mark.asyncio class TestWarmClingenCacheUnit: @@ -49,26 +97,15 @@ async def test_warms_cache_for_variants_with_caids( """Job calls get_clingen_allele_data for each distinct allele ID.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) - # Create two variants with the same CAID — should only warm once (distinct) - for i, caid in enumerate(["CA111111", "CA222222", "CA111111"]): - variant = Variant( - urn=f"urn:variant:warm-test-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 1}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 1}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + # Three variants, two sharing the same CAID — should only warm 2 distinct IDs. + # Two separate allele rows are needed (different VRS digests) to get 2 distinct CAIDs. + _make_allele_with_caid(session, score_set.id, "warm-test-0", "CA111111", "digest-warm-0") + _make_allele_with_caid(session, score_set.id, "warm-test-1", "CA222222", "digest-warm-1") + # Third variant points to same allele as first (same CAID CA111111 via a fresh allele row + # with the same digest — but since get_alleles_for_score_set returns per-variant rows and + # caid dedup happens in the warmer, we can share the digest). + _make_allele_with_caid(session, score_set.id, "warm-test-2", "CA111111", "digest-warm-2") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -100,24 +137,8 @@ async def test_skips_null_and_multi_variant_caids( caids = ["CA333333", None, "CA-MULTI-001,CA-MULTI-002"] for i, caid in enumerate(caids): - variant = Variant( - urn=f"urn:variant:warm-filter-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 10}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 10}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-filter-{i}", caid, f"digest-filter-{i}") + session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) @@ -147,24 +168,8 @@ async def test_continues_on_individual_fetch_failure( score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) for i, caid in enumerate(["CA444444", "CA555555"]): - variant = Variant( - urn=f"urn:variant:warm-fail-{i}", - score_set_id=score_set.id, - hgvs_nt=f"NM_000000.1:c.{i + 20}A>G", - hgvs_pro=f"NP_000000.1:p.Met{i + 20}Val", - data={}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=caid, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + _make_allele_with_caid(session, score_set.id, f"warm-fail-{i}", caid, f"digest-fail-{i}") + session.commit() # First call raises, second succeeds mock_get_allele_data = AsyncMock( @@ -192,7 +197,7 @@ async def test_only_warms_current_mapped_variants( with_warm_clingen_cache_job, sample_warm_clingen_cache_job_run, ): - """Job only fetches allele IDs from current (not superseded) mapped variants.""" + """Job only fetches allele IDs from current (not superseded) mapping records.""" score_set = session.get(ScoreSet, sample_warm_clingen_cache_job_run.job_params["score_set_id"]) variant = Variant( @@ -203,25 +208,65 @@ async def test_only_warms_current_mapped_variants( data={}, ) session.add(variant) - session.commit() + session.flush() - # Non-current mapped variant should be ignored - old_mv = MappedVariant( - variant_id=variant.id, + # Superseded allele (MappingRecord with valid_to set — not current) + old_allele = Allele( + vrs_digest="digest-old-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA666666", - current=False, - mapped_date="2023-01-01T00:00:00Z", - mapping_api_version="0.9.0", ) - # Current mapped variant should be included - current_mv = MappedVariant( + session.add(old_allele) + session.flush() + + from datetime import datetime, timezone + + old_mapping_record = MappingRecord( variant_id=variant.id, + assay_level="genomic", + mapping_api_version="0.9.0", + mapped_date=date(2023, 1, 1), + ) + # Close this record so it is non-current (valid_to set) + old_mapping_record.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_mapping_record) + session.flush() + + old_link = MappingRecordAllele( + mapping_record_id=old_mapping_record.id, + allele_id=old_allele.id, + is_authoritative=True, + ) + old_link.valid_to = datetime(2024, 1, 1, tzinfo=timezone.utc) + session.add(old_link) + session.flush() + + # Current allele (MappingRecord with valid_to=NULL — current) + current_allele = Allele( + vrs_digest="digest-current-mv", + level="genomic", + post_mapped={"type": "Allele"}, clingen_allele_id="CA777777", - current=True, - mapped_date="2024-01-01T00:00:00Z", + ) + session.add(current_allele) + session.flush() + + current_mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", mapping_api_version="1.0.0", + mapped_date=date.today(), + ) + session.add(current_mapping_record) + session.flush() + + current_link = MappingRecordAllele( + mapping_record_id=current_mapping_record.id, + allele_id=current_allele.id, + is_authoritative=True, ) - session.add_all([old_mv, current_mv]) + session.add(current_link) session.commit() mock_get_allele_data = AsyncMock(return_value={"some": "data"}) From ac4cfa920bde621a0468a4134f82b40d640e0ae8 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 24/93] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- .../worker/jobs/external_services/clingen.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 27aa1b962..42cdcb36a 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -121,14 +121,12 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_car", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_car", + "resource": score_set.urn, + "correlation_id": correlation_id, + }) job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) @@ -480,14 +478,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_ldh", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_ldh", + "resource": score_set.urn, + "correlation_id": correlation_id, + }) job_manager.update_progress(0, 100, "Starting LDH mapped resource submission.") logger.info(msg="Started LDH mapped resource submission", extra=job_manager.logging_context()) @@ -560,12 +556,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: loop = asyncio.get_running_loop() submission_successes, submission_failures = await loop.run_in_executor(ctx["pool"], blocking) job_manager.update_progress(90, 100, "Finalizing LDH mapped resource submission.") - job_manager.save_to_context( - { - "ldh_submission_successes": len(submission_successes), - "ldh_submission_failures": len(submission_failures), - } - ) + job_manager.save_to_context({ + "ldh_submission_successes": len(submission_successes), + "ldh_submission_failures": len(submission_failures), + }) # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) From dba7bfebcaa877ac7954e5468cc2c3d77ef27e4d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 11:45:14 -0700 Subject: [PATCH 25/93] feat(gnomad): migrate gnomAD linkage to the Allele model with version-keyed refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace MappedVariant-based gnomAD linkage with valid-time GnomadAlleleLink rows keyed on the deduplicated Allele. Linking covers every current allele of a score set (authoritative and RT-derived), so protein/coding score sets — whose genomic allele is RT-derived — are no longer dropped; per-variant annotation status flows through the _annotate_gnomad choke point against authoritative links only (interim bandaid, the AnnotationEvent migration seam). Refresh / idempotency model: - One live link per allele (unique index on allele_id); a gnomAD version bump supersedes rather than accumulating one live link per version. - Supersede only on change — an unchanged re-run leaves the live link untouched, so the valid-time history records no spurious boundary. - Version-keyed skip avoids re-fetching alleles already current at the version; a force param bypasses the skip (re-ingestion / heal) without churning unchanged links. - Per-variant status is a per-run audit event: created / preexisting / skipped. Normalize CAIDs across the Athena join: the gnomAD Hail dump drops leading zeros (CA025094 -> CA25094), so exact-string matching silently missed zero-padded CAIDs. Extract group_alleles_for_annotation as the shared allele-grouping primitive for allele-subject annotation jobs (adopted by gnomAD; VEP/ClinVar to follow). Refs #742. Partially addresses #722 (CAID-completeness tracking there stays open). --- .../e5f7a9c1b3d4_add_gnomad_allele_links.py | 71 ++++ src/mavedb/lib/clingen/alleles.py | 56 +++- src/mavedb/lib/gnomad.py | 197 ++++++----- src/mavedb/models/__init__.py | 1 + src/mavedb/models/gnomad_allele_link.py | 60 ++++ src/mavedb/models/gnomad_variant.py | 13 +- .../worker/jobs/external_services/gnomad.py | 215 +++++++----- tests/lib/clingen/test_alleles.py | 69 ++++ tests/lib/test_gnomad.py | 309 +++++++++--------- tests/worker/jobs/conftest.py | 85 +++++ .../jobs/external_services/test_gnomad.py | 245 +++++++++++--- 11 files changed, 949 insertions(+), 372 deletions(-) create mode 100644 alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py create mode 100644 src/mavedb/models/gnomad_allele_link.py create mode 100644 tests/lib/clingen/test_alleles.py diff --git a/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py new file mode 100644 index 000000000..54f3e3d58 --- /dev/null +++ b/alembic/versions/e5f7a9c1b3d4_add_gnomad_allele_links.py @@ -0,0 +1,71 @@ +"""add gnomad_allele_links table + +Revision ID: e5f7a9c1b3d4 +Revises: d4e6f8a0b2c3 +Create Date: 2026-06-18 + +New valid-time link table connecting deduplicated alleles to gnomAD variants, replacing the frozen +gnomad_variants_mapped_variants association for new-model writes (Step 1 of the annotation +infrastructure migration, docs/design/annotation-infrastructure-migration.md). A link is live while +valid_to is NULL; the partial unique index enforces a single live link per (allele, gnomad variant) +pair. The existing gnomad_variants_mapped_variants table is left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e5f7a9c1b3d4" +down_revision = "d4e6f8a0b2c3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "gnomad_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("gnomad_variant_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_gnomad_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["gnomad_variant_id"], + ["gnomad_variants.id"], + name="fk_gnomad_allele_links_gnomad_variant_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_gnomad_allele_links_allele_id", + "gnomad_allele_links", + ["allele_id"], + ) + op.create_index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_allele_links", + ["gnomad_variant_id"], + ) + # One live link per allele: gnomAD frequency is a single current value, so a version bump + # supersedes the prior link rather than accumulating one live link per version. + op.create_index( + "uq_gnomad_allele_links_live", + "gnomad_allele_links", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_gnomad_allele_links_live", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_gnomad_variant_id", table_name="gnomad_allele_links") + op.drop_index("ix_gnomad_allele_links_allele_id", table_name="gnomad_allele_links") + op.drop_table("gnomad_allele_links") diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 527572233..79baf47ff 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -5,7 +5,8 @@ score set. A single definition here prevents the two jobs from drifting apart. """ -from typing import NamedTuple +from dataclasses import dataclass, field +from typing import Callable, Generic, Iterable, NamedTuple, Optional, TypeVar from sqlalchemy import select from sqlalchemy.orm import Session @@ -15,6 +16,8 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant +P = TypeVar("P") + class ScoreSetAlleleRow(NamedTuple): """One (allele, variant) link for a score set. An allele shared by multiple variants @@ -59,3 +62,54 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] + + +@dataclass +class AlleleAnnotationGroup(Generic[P]): + """One allele's annotation work-unit: a job-specific ``payload`` plus the variants that receive a + per-variant annotation-status (VAS) row for it. + + ``authoritative_variant_ids`` is the interim **bandaid seam** (see + docs/design/allele-annotation-status.md). An annotation is an allele-level fact, but VAS is + per-variant; fanning it to every variant the allele backs would write multiple ``current`` rows + for one variant. So status is fanned only to the variants for which this allele is the + *authoritative* measurement. At the AnnotationEvent migration the per-variant fan-out goes away + and this list is no longer needed. The allele is still grouped (and linked/annotated at the + allele level) regardless — a purely RT-derived allele simply has an empty list here. + """ + + payload: P + authoritative_variant_ids: list[int] = field(default_factory=list) + + +def group_alleles_for_annotation( + rows: Iterable[ScoreSetAlleleRow], + payload: Callable[[ScoreSetAlleleRow], Optional[P]], +) -> dict[int, AlleleAnnotationGroup[P]]: + """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one + work-unit per allele, keyed by ``allele_id``. + + The same allele recurs once per variant that links it (and can be authoritative for one variant + while RT-derived for another). This groups those rows so every annotation job shares one shape: + one entry per allele, carrying the authoritative-variant fan-out set for the bandaid. + + ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for + gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it + carries no CAID), replacing each job's ad-hoc ``if row.x is None: continue``. ``payload`` must be + a pure function of allele-level fields so its result is stable across an allele's rows. + + Grouping on ``allele_id`` rather than ``vrs_digest`` is intentional: the two are 1:1 (content + addressing makes ``vrs_digest`` unique), so the groups are identical either way, and ``allele_id`` + is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry + it on the row and store against it — the grouping contract here does not change. + """ + groups: dict[int, AlleleAnnotationGroup[P]] = {} + for row in rows: + if row.allele_id not in groups: + built = payload(row) + if built is None: + continue + groups[row.allele_id] = AlleleAnnotationGroup(payload=built) + if row.is_authoritative: + groups[row.allele_id].authoritative_variant_ids.append(row.variant_id) + return groups diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9bfa0fec9..8456ec35d 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -3,19 +3,24 @@ import re from typing import Any, Sequence, Union -from sqlalchemy import Connection, Row, select, text +from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session -from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import batched -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant GNOMAD_DB_NAME = "gnomAD" -GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") # e.g., "v4.1" +GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") +_CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" +""" +Strip leading zeros from a CAID's numeric portion, keeping at least one digit. +Kept byte-for-byte in sync with the SQL form used to normalize Allele.clingen_allele_id +in link_gnomad_variants_to_alleles. +""" + logger = logging.getLogger(__name__) @@ -46,6 +51,18 @@ def gnomad_table_name() -> str: return table_name +def normalize_caid(caid: str) -> str: + """Normalize a ClinGen CAID by stripping leading zeros from its numeric portion. + + The gnomAD Hail/Athena dump drops leading zeros from CAIDs — MaveDB's ``CA025094`` is recorded as + ``CA25094`` — so an exact-string join silently misses every zero-padded CAID (issue #722). Both + sides of the join are normalized to the unpadded form to repair the match. ``CA025094`` and + ``CA25094`` denote the same ClinGen allele, so this can never collide distinct alleles. A value + that is not a recognizable CAID (no ``CA`` prefix + digits) is returned unchanged. + """ + return re.sub(_CAID_LEADING_ZERO_RE, r"\1\2", caid) + + def allele_list_from_list_like_string(alleles_string: str) -> list[str]: """ Convert a list-like string representation of alleles into a Python list. @@ -94,8 +111,9 @@ def gnomad_variant_data_for_caids( Raises: sqlalchemy.exc.SQLAlchemyError: If there is an error executing the query. """ + # Normalize to the unpadded form the dump stores so the IN-list matches zero-padded CAIDs (see issue #722). chunked_caids = batched(caids, 16250) - caid_strs = [",".join(f"'{caid}'" for caid in chunk) for chunk in chunked_caids] + caid_strs = [",".join(f"'{normalize_caid(caid)}'" for caid in chunk) for chunk in chunked_caids] save_to_logging_context({"num_caids": len(caids), "num_chunks": len(caid_strs)}) result_rows: list[Row[Any]] = [] @@ -132,31 +150,40 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_mapped_variants( - db: Session, gnomad_variant_data: Sequence[Row[Any]], only_current: bool = True -) -> int: - """ - Links gnomAD variants to mapped variants in the database based on CAIDs. Note that this function does - not commit this data to the database; it only prepares the relationships. +def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[Row[Any]]) -> set[int]: + """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. - Args: - caids (list[str]): A list of CAIDs to link with gnomAD variants. + Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a + valid-time :class:`GnomadAlleleLink`, so one gnomAD variant fans out to every allele sharing the + CAID (cross-score-set dedup included). Each allele holds at most one live link, superseded **only + on change**: a live link already pointing to the resolved variant is left untouched (an unchanged + re-run writes no spurious valid-time boundary); a new/different/older-version target retires it and + inserts a successor, keyed on ``allele_id`` so a version bump replaces rather than accumulates. The + guard is load-bearing despite the job's upstream skip — shared CAIDs and ``force`` runs still reach + it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is + logged and superseded newest-wins, not raised. + + Does not commit. Returns the allele ids whose link was created or superseded this run. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) - save_to_logging_context({"only_current": only_current}) - logger.debug(msg="Linking gnomAD variants to mapped variants", extra=logging_context()) + logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) - linked_gnomad_variants = 0 - annotation_manager = AnnotationStatusManager(db) + changed_allele_ids: set[int] = set() for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - mapped_variants_with_caids_query = select(MappedVariant).where(MappedVariant.clingen_allele_id == row.caid) - if only_current: - mapped_variants_with_caids_query = mapped_variants_with_caids_query.where(MappedVariant.current.is_(True)) - mapped_variants_with_caids = db.scalars(mapped_variants_with_caids_query).all() + # Match on the unpadded CAID: the dump's caid is already stripped, while the stored CAID may + # be zero-padded, so normalize both sides (issue #722). regexp_replace mirrors normalize_caid. + alleles_with_caid = db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2") + == normalize_caid(row.caid) + ) + ).all() + if not alleles_with_caid: + continue gnomad_identifier_for_variant = gnomad_identifier( row.__getattribute__("locus.contig"), @@ -172,80 +199,86 @@ def link_gnomad_variants_to_mapped_variants( if faf95_max is not None: faf95_max = float(faf95_max) - for mapped_variant in mapped_variants_with_caids: - # Remove any existing gnomAD variants for this mapped variant that match the current gnomAD data version to avoid data duplication. - # There should only be one gnomAD variant per mapped variant per gnomAD data version, since each gnomAD variant can only match to one - # CAID. - for linked_gnomad_variant in mapped_variant.gnomad_variants: - if linked_gnomad_variant.db_version == GNOMAD_DATA_VERSION: - logger.debug( - msg=f"Removing existing gnomAD variant {linked_gnomad_variant.db_identifier} from mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", - extra=logging_context(), - ) - mapped_variant.gnomad_variants.remove(linked_gnomad_variant) - - existing_gnomad_variant = db.scalar( - select(GnomADVariant).where( - GnomADVariant.db_name == "gnomAD", - GnomADVariant.db_identifier == gnomad_identifier_for_variant, - GnomADVariant.db_version == GNOMAD_DATA_VERSION, - ) + # One gnomAD variant per (identifier, version): get-or-create so repeated CAIDs and re-runs + # reuse the same row. Flush so a freshly created variant has an id for the link below. + gnomad_variant = db.scalar( + select(GnomADVariant).where( + GnomADVariant.db_name == GNOMAD_DB_NAME, + GnomADVariant.db_identifier == gnomad_identifier_for_variant, + GnomADVariant.db_version == GNOMAD_DATA_VERSION, + ) + ) + if gnomad_variant is None: + logger.debug( + msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), + ) + gnomad_variant = GnomADVariant( + db_name=GNOMAD_DB_NAME, + db_identifier=gnomad_identifier_for_variant, + db_version=GNOMAD_DATA_VERSION, + allele_count=allele_count, + allele_number=allele_number, + allele_frequency=allele_frequency, # type: ignore + faf95_max_ancestry=faf95_max_ancestry, + faf95_max=faf95_max, # type: ignore + ) + db.add(gnomad_variant) + db.flush() + else: + logger.debug( + msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + extra=logging_context(), ) - if existing_gnomad_variant is None: - logger.debug( - msg=f"Creating new gnomAD variant for identifier {gnomad_identifier_for_variant}", - extra=logging_context(), + for allele in alleles_with_caid: + live_link = db.scalar( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, ) - gnomad_variant = GnomADVariant( - db_name=GNOMAD_DB_NAME, - db_identifier=gnomad_identifier_for_variant, - db_version=GNOMAD_DATA_VERSION, - allele_count=allele_count, - allele_number=allele_number, - allele_frequency=allele_frequency, # type: ignore - faf95_max_ancestry=faf95_max_ancestry, - faf95_max=faf95_max, # type: ignore - ) - else: - logger.debug( - msg=f"Found existing gnomAD variant for identifier {gnomad_identifier_for_variant}", + ) + + # No change: live link already points here — leave it untouched (no spurious boundary). + if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + continue + + if ( + live_link is not None + and live_link.gnomad_variant.db_version == GNOMAD_DATA_VERSION + and live_link.gnomad_variant.db_identifier != gnomad_identifier_for_variant + ): + logger.warning( + msg=( + f"CAID {allele.clingen_allele_id} for allele {allele.id} resolved to " + f"{gnomad_identifier_for_variant} at version {GNOMAD_DATA_VERSION}, but a live link " + f"already points to {live_link.gnomad_variant.db_identifier} at the same version. " + "Superseding (newest wins); investigate the gnomAD source for a re-resolved CAID." + ), extra=logging_context(), ) - gnomad_variant = existing_gnomad_variant - if gnomad_variant not in mapped_variant.gnomad_variants: - mapped_variant.gnomad_variants.append(gnomad_variant) - linked_gnomad_variants += 1 - - db.add(gnomad_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "gnomad_db_identifier": gnomad_variant.db_identifier, - } - }, - current=True, + # Change: retire any live link for the allele, insert the successor (allele-keyed, so a + # version bump replaces rather than accumulates). + GnomadAlleleLink.supersede_live_where( + db, + [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], + GnomadAlleleLink.allele_id == allele.id, ) + changed_allele_ids.add(allele.id) logger.debug( - msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to mapped variant {mapped_variant.id} ({mapped_variant.clingen_allele_id})", + msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", extra=logging_context(), ) logger.info( - f"Linked {len(mapped_variants_with_caids)} mapped variants with CAID {row.caid} to gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" + f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - annotation_manager.flush() - - save_to_logging_context({"linked_gnomad_variants": linked_gnomad_variants}) + save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) logger.info( - msg=f"Linked a total of {linked_gnomad_variants} gnomAD variants to mapped variants.", + msg=f"Created or superseded {len(changed_allele_ids)} allele links this run.", extra=logging_context(), ) - return linked_gnomad_variants + return changed_allele_ids diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 35c1ba0a5..5284da11b 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -11,6 +11,7 @@ "experiment", "experiment_set", "genome_identifier", + "gnomad_allele_link", "gnomad_variant", "job_dependency", "job_run", diff --git a/src/mavedb/models/gnomad_allele_link.py b/src/mavedb/models/gnomad_allele_link.py new file mode 100644 index 000000000..75294ba66 --- /dev/null +++ b/src/mavedb/models/gnomad_allele_link.py @@ -0,0 +1,60 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .gnomad_variant import GnomADVariant + + +class GnomadAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a gnomAD variant. + + Replaces the frozen ``gnomad_variants_mapped_variants`` association table for new-model writes. + A link is live while ``valid_to`` is NULL; a gnomAD version bump retires the live row and inserts + a successor rather than deleting, so prior-version frequency links remain queryable point-in-time. + The partial unique index enforces **a single live link per allele** — gnomAD frequency is one + current value, so a new version supersedes the old (unlike ClinVar, which keeps one live link per + release). This matches the VEP consequence shape, not the ClinVar control shape. + """ + + __tablename__ = "gnomad_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + gnomad_variant_id: Mapped[int] = Column( + Integer, + ForeignKey("gnomad_variants.id", ondelete="RESTRICT"), + nullable=False, + ) + + allele: Mapped["Allele"] = relationship("Allele") + gnomad_variant: Mapped["GnomADVariant"] = relationship("GnomADVariant", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_gnomad_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_gnomad_allele_links_gnomad_variant_id", + "gnomad_variant_id", + ), + # At most one live link per allele. A version bump supersedes (retires the old, inserts the + # new) rather than accumulating per-version live links; superseded rows stay for point-in-time + # queries. Only the live row participates in this constraint. + Index( + "uq_gnomad_allele_links_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index 0f4a00a50..bfc7c8026 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,20 +1,21 @@ from datetime import date from typing import TYPE_CHECKING -from sqlalchemy import Column, Date, Integer, Float, String +from sqlalchemy import Column, Date, Float, Integer, String from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: + from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.mapped_variant import MappedVariant class GnomADVariant(Base): __tablename__ = "gnomad_variants" - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) db_name = Column(String, nullable=False) db_identifier = Column(String, nullable=False, index=True) @@ -30,8 +31,16 @@ class GnomADVariant(Base): creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen association to the old MappedVariant model — read by serving for existing data, never + # written for new score sets (which link through ``allele_links`` instead). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=gnomad_variants_mapped_variants_association_table, back_populates="gnomad_variants", ) + + # Valid-time links to deduplicated alleles (new-model writes). + allele_links: Mapped[list["GnomadAlleleLink"]] = relationship( + "GnomadAlleleLink", + back_populates="gnomad_variant", + ) diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 969c3a20e..8daa5c9af 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -1,29 +1,29 @@ """gnomAD variant linking jobs for population frequency annotation. -This module handles linking of mapped variants to gnomAD (Genome Aggregation Database) +This module handles linking of deduplicated alleles to gnomAD (Genome Aggregation Database) variants to provide population frequency and other genomic context information. This enrichment helps researchers understand the clinical significance and rarity of variants in their datasets. """ import logging -from typing import Sequence from sqlalchemy import select from mavedb.db import athena from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, gnomad_variant_data_for_caids, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -31,17 +31,51 @@ logger = logging.getLogger(__name__) +def _annotate_gnomad( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a GNOMAD_ALLELE_FREQUENCY annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for gnomAD's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (which variant's mapping drove the linkage). See + docs/design/allele-annotation-status.md. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + version=GNOMAD_DATA_VERSION, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + @with_pipeline_management async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: """ - Link mapped variants to gnomAD variants based on ClinGen Allele IDs (CAIDs). - This job fetches mapped variants associated with a given score set that have CAIDs, - retrieves corresponding gnomAD variant data, and establishes links between them - in the database. + Link deduplicated alleles to gnomAD variants based on ClinGen Allele IDs (CAIDs). + This job fetches the current authoritative alleles of a score set that carry CAIDs, + retrieves corresponding gnomAD variant data, and establishes valid-time links between them. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants to process. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the version-keyed skip and re-fetch every CAID-bearing + allele. The linker still supersedes only on change, so a forced re-run of unchanged data + writes no new links. Use for re-ingestion or to heal suspected link corruption. Args: ctx (dict): The job context dictionary. @@ -49,10 +83,10 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager (JobManager): The job manager instance for database and logging operations. Side Effects: - - Updates MappedVariant records to link to gnomAD variants. + - Creates GnomadAlleleLink rows linking alleles to gnomAD variants. Returns: - dict: Result indicating success and any exception details + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ # Get the job definition we are working on job = job_manager.get_job() @@ -63,6 +97,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] # Setup initial context and progress job_manager.save_to_context( @@ -76,87 +111,117 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # We filter out mapped variants that do not have a CAID, so this query is typed # as a Sequence[str]. Ignore MyPy's type checking here. - variant_caids: Sequence[str] = job_manager.db.scalars( - select(MappedVariant.clingen_allele_id) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), - ) - ).all() # type: ignore + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the + # RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=lambda row: row.clingen_allele_id, + ) - num_variant_caids = len(variant_caids) - job_manager.save_to_context({"num_variants_to_link_gnomad": num_variant_caids}) + num_alleles_with_caids = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) - if not variant_caids: + if not allele_data: logger.warning( - msg="No current mapped variants with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", + msg="No current alleles with CAIDs were found for this score set. Skipping gnomAD linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"linked_count": 0, "skipped_count": 0}) + return JobExecutionOutcome.succeeded( + data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} + ) - job_manager.update_progress(10, 100, f"Found {num_variant_caids} variants with CAIDs to link to gnomAD variants.") - logger.info( - msg="Found current mapped variants with CAIDs for this score set. Attempting to link them to gnomAD variants.", - extra=job_manager.logging_context(), + job_manager.update_progress( + 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." ) - # Fetch gnomAD variant data for the CAIDs - with athena.engine.connect() as athena_session: - logger.debug("Fetching gnomAD variants from Athena.") - gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - - num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live gnomAD link at the current gnomAD version.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(GnomadAlleleLink.allele_id) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.current) + .where(GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ).all() + ) - # NOTE: Proceed intentionally with linking even if no matches were found, to record skipped annotations. + # Cost: skip alleles already linked at the current version (they can't change). force re-fetches + # all; the linker still supersedes only on change, so a forced no-op writes nothing. + already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) + variant_caids = sorted({allele_data[aid].payload for aid in allele_data if aid not in already_current}) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_caids_to_query": len(variant_caids), + "force": force, + } + ) - job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) - job_manager.update_progress(75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs.") + changed_allele_ids: set[int] = set() + if variant_caids: + with athena.engine.connect() as athena_session: + logger.debug("Fetching gnomAD variants from Athena.") + gnomad_variant_data = gnomad_variant_data_for_caids(athena_session, variant_caids) - # Link mapped variants to gnomAD variants - logger.info(msg="Attempting to link mapped variants to gnomAD variants.", extra=job_manager.logging_context()) - num_linked_gnomad_variants = link_gnomad_variants_to_mapped_variants(job_manager.db, gnomad_variant_data) - job_manager.db.flush() + num_gnomad_variants_with_caid_match = len(gnomad_variant_data) + job_manager.save_to_context({"num_gnomad_variants_with_caid_match": num_gnomad_variants_with_caid_match}) + job_manager.update_progress( + 75, 100, f"Found {num_gnomad_variants_with_caid_match} gnomAD variants matching CAIDs." + ) - # For variants which are not linked, create annotation status records indicating skipped linkage - mapped_variants_with_caids = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .join(ScoreSet) - .where( - ScoreSet.urn == score_set.urn, - MappedVariant.current.is_(True), - MappedVariant.clingen_allele_id.is_not(None), + logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) + changed_allele_ids = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + job_manager.db.flush() + else: + logger.info( + msg="All CAID-bearing alleles are already linked at the current gnomAD version; skipping Athena query.", + extra=job_manager.logging_context(), ) - ).all() + job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") + + # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting + # if already current, SKIPPED if no current-version link (gnomAD had no data for the CAID). + current_after = alleles_linked_at_current_version(set(allele_data.keys())) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in mapped_variants_with_caids: - if not mapped_variant.gnomad_variants: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=AnnotationStatus.SKIPPED, + created_count = preexisting_count = skipped_count = 0 + for allele_id, entry in allele_data.items(): + if allele_id in current_after: + action = "created" if allele_id in changed_allele_ids else "preexisting" + if action == "created": + created_count += 1 + else: + preexisting_count += 1 + _annotate_gnomad( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": entry.payload, "action": action}, + ) + else: + skipped_count += 1 + _annotate_gnomad( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No gnomAD variant could be linked for this mapped variant.", - }, - current=True, + error_message="No gnomAD variant could be linked for this allele.", + metadata={"clingen_allele_id": entry.payload}, ) annotation_manager.flush() - # Save final context and progress - job_manager.save_to_context({"num_mapped_variants_linked_to_gnomad_variants": num_linked_gnomad_variants}) - logger.info(msg="Done linking gnomAD variants to mapped variants.", extra=job_manager.logging_context()) + outcome_data = { + "created_allele_count": created_count, + "preexisting_allele_count": preexisting_count, + "skipped_allele_count": skipped_count, + } + job_manager.save_to_context(outcome_data) + logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "linked_count": num_linked_gnomad_variants, - "skipped_count": num_variant_caids - num_linked_gnomad_variants, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py new file mode 100644 index 000000000..fbcfaa986 --- /dev/null +++ b/tests/lib/clingen/test_alleles.py @@ -0,0 +1,69 @@ +"""Unit tests for the pure allele-grouping primitive shared by the annotation jobs.""" + +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation + + +def _row(allele_id, variant_id, *, is_authoritative, caid="CA1"): + return ScoreSetAlleleRow( + allele_id=allele_id, + post_mapped={"type": "Allele"}, + clingen_allele_id=caid, + variant_id=variant_id, + is_authoritative=is_authoritative, + ) + + +def test_collapses_rows_to_one_group_per_allele(): + rows = [ + _row(1, 10, is_authoritative=True), + _row(1, 11, is_authoritative=True), + _row(2, 12, is_authoritative=True, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {1, 2} + assert groups[1].payload == "CA1" + assert groups[1].authoritative_variant_ids == [10, 11] + assert groups[2].authoritative_variant_ids == [12] + + +def test_only_authoritative_links_contribute_variant_ids(): + """An allele that is authoritative for one variant and RT-derived for another fans status only to + the authoritative variant — the bandaid invariant.""" + rows = [ + _row(1, 10, is_authoritative=True), + _row(1, 11, is_authoritative=False), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert groups[1].authoritative_variant_ids == [10] + + +def test_purely_rt_derived_allele_is_grouped_with_empty_fan_out(): + """A non-authoritative-only allele is still grouped (so it gets linked/annotated at the allele + level) but contributes no per-variant VAS row.""" + rows = [_row(1, 10, is_authoritative=False)] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {1} + assert groups[1].authoritative_variant_ids == [] + + +def test_payload_returning_none_skips_the_allele(): + """Returning None from payload drops the allele entirely — replacing each job's ad-hoc + 'no CAID / no HGVS -> continue' filter.""" + rows = [ + _row(1, 10, is_authoritative=True, caid=None), + _row(2, 11, is_authoritative=True, caid="CA2"), + ] + + groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) + + assert set(groups) == {2} + + +def test_empty_rows_yield_empty_grouping(): + assert group_alleles_for_annotation([], payload=lambda r: r.clingen_allele_id) == {} diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index 14dde9527..e00b57e76 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -3,8 +3,7 @@ from unittest.mock import patch import pytest - -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from sqlalchemy import select pyathena = pytest.importorskip("pyathena") fastapi = pytest.importorskip("fastapi") @@ -13,14 +12,15 @@ allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, - link_gnomad_variants_to_mapped_variants, + link_gnomad_variants_to_alleles, + normalize_caid, ) +from mavedb.models.allele import Allele +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, - TEST_MINIMAL_MAPPED_VARIANT, ) ### Tests for gnomad_identifier function ### @@ -77,6 +77,23 @@ def test_gnomad_table_name_raises_if_env_not_set(): gnomad_table_name() +### Tests for normalize_caid function ### + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("CA025094", "CA25094"), # the #722 example: gnomAD dump drops the leading zero + ("CA000123", "CA123"), # multiple leading zeros collapse + ("CA341478553", "CA341478553"), # already unpadded — unchanged + ("CA0", "CA0"), # keep the final digit even if it is a zero + ("not-a-caid", "not-a-caid"), # unrecognized input is passed through + ], +) +def test_normalize_caid(raw, expected): + assert normalize_caid(raw) == expected + + ### Tests for allele_list_from_list_like_string function ### @@ -118,217 +135,185 @@ def test_allele_list_from_list_like_string_invalid_format_not_list(): # If the package is working correctly, this function should work as expected. -### Tests for link_gnomad_variants_to_mapped_variants function ### +### Tests for link_gnomad_variants_to_alleles function ### -def _verify_annotation_status(session, mapped_variants, expected_version): - annotations = session.query(VariantAnnotationStatus).all() - assert len(annotations) == len(mapped_variants) +def _make_allele(session, caid, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele carrying a CAID.""" + allele = Allele(vrs_digest=vrs_digest, level=level, clingen_allele_id=caid) + session.add(allele) + session.commit() + session.refresh(allele) + return allele - for mapped_variant, annotation in zip(mapped_variants, annotations): - assert annotation.variant_id == mapped_variant.variant_id - assert annotation.annotation_type == "gnomad_allele_frequency" - assert annotation.version == expected_version +def _live_links_for(session, allele_id): + return session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.current, + ) + ).all() -def test_links_new_gnomad_variant_to_mapped_variant( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 - session.commit() +def _assert_gnomad_variant_matches(gnomad_variant, **overrides): + expected = TEST_GNOMAD_VARIANT.copy() + expected.pop("creation_date") + expected.pop("modification_date") + expected.update(overrides) + for attr, value in expected.items(): + assert getattr(gnomad_variant, attr) == value - session.refresh(mapped_variant) - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") +def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} + session.commit() - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant) -def test_can_link_gnomad_variants_with_none_type_faf_fields( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() +def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnomad_variant_row): + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max_gen_anc", None) mocked_gnomad_variant_row.__setattr__("joint.fafmax.faf95_max", None) with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - gnomad_variant_comparator["faf95_max"] = None - gnomad_variant_comparator["faf95_max_ancestry"] = None - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + _assert_gnomad_variant_matches(live_links[0].gnomad_variant, faf95_max=None, faf95_max_ancestry=None) -def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): +def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): gnomad_variant = GnomADVariant(**TEST_GNOMAD_VARIANT) - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) session.add(gnomad_variant) session.commit() + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") - - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - - _verify_annotation_status(session, [mapped_variant], TEST_GNOMAD_DATA_VERSION) - + # Reused the existing gnomAD variant rather than creating a second. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant_id == gnomad_variant.id -def test_adding_existing_gnomad_variant_with_same_version_does_not_result_in_duplication( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant = setup_lib_db_with_mapped_variant - mapped_variant.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant) - session.commit() - with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 +def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): + """Supersede only on change: a second run with identical data writes nothing — one live link, + no retired rows, so the valid-time history records no spurious boundary.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() + # Second run sees the live link already points to this gnomAD variant → no change reported. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == set() session.commit() - session.refresh(mapped_variant) - - edited_saved_gnomad_variant = TEST_GNOMAD_VARIANT.copy() - edited_saved_gnomad_variant.pop("creation_date") - edited_saved_gnomad_variant.pop("modification_date") + # One link, still live, never retired — the re-run did not churn the history. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 1 + assert all_links[0].valid_to is None + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - assert len(mapped_variant.gnomad_variants) == 1 - for attr in edited_saved_gnomad_variant: - assert getattr(mapped_variant.gnomad_variants[0], attr) == edited_saved_gnomad_variant[attr] - _verify_annotation_status(session, [mapped_variant, mapped_variant], TEST_GNOMAD_DATA_VERSION) +def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_variant_row): + """A new gnomAD version retires the prior link and installs the new one — exactly one live link + per allele (not one per version), with the old version preserved as a retired row.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() -def test_links_multiple_rows_and_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + session.commit() - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_version == "v2.new" + # Old-version link retired, not deleted; both gnomAD variant rows persist. + all_links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(all_links) == 2 + assert len([link for link in all_links if link.valid_to is not None]) == 1 + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + + +def test_same_version_different_identifier_supersedes_newest_wins(session, mocked_gnomad_variant_row): + """A CAID re-resolving to a different identifier within the same version is an anomaly: log and + supersede newest-wins rather than raise — one odd allele must not abort the batch.""" + allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + # Prior live link at the current version, but to a different identifier than the row resolves to. + stale = GnomADVariant( + db_name="gnomAD", + db_identifier="9-99999-C-T", + db_version=TEST_GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(stale) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=stale.id)) session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 2 + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 1 - assert len(mapped_variant2.gnomad_variants) == 1 - for mv in [mapped_variant1, mapped_variant2]: - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] - - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) - + live_links = _live_links_for(session, allele.id) + assert len(live_links) == 1 + assert live_links[0].gnomad_variant.db_identifier != "9-99999-C-T" # newest wins -def test_returns_zero_when_no_mapped_variants(session, mocked_gnomad_variant_row): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 0 - _verify_annotation_status(session, [], TEST_GNOMAD_DATA_VERSION) - - -def test_only_current_flag_filters_variants(session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mocked_gnomad_variant_row): + """A CAID shared by multiple alleles (cross-score-set dedup) fans the gnomAD variant to each.""" + allele1 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") + allele2 = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-2", level="cdna") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row]) - assert result == 1 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele1.id, allele2.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - - assert len(mapped_variant1.gnomad_variants) == 0 - assert len(mapped_variant2.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mapped_variant2.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + for allele in (allele1, allele2): + assert len(_live_links_for(session, allele.id)) == 1 + # Both links point at the single get-or-created gnomAD variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 1 - _verify_annotation_status(session, [mapped_variant2], TEST_GNOMAD_DATA_VERSION) - -def test_only_current_flag_is_false_operates_on_all_variants( - session, mocked_gnomad_variant_row, setup_lib_db_with_mapped_variant -): - mapped_variant1 = setup_lib_db_with_mapped_variant - mapped_variant2 = MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_variant1.variant_id) - - mapped_variant1.current = False - mapped_variant1.clingen_allele_id = mocked_gnomad_variant_row.caid - mapped_variant2.clingen_allele_id = mocked_gnomad_variant_row.caid - session.add(mapped_variant1) - session.add(mapped_variant2) - session.commit() +def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gnomad_variant_row): + """The gnomAD dump records CAIDs without leading zeros (#722). An allele stored with the + zero-padded CAID must still match the dump's stripped form across the join.""" + allele = _make_allele(session, "CA025094", vrs_digest="vrs-1") + mocked_gnomad_variant_row.caid = "CA25094" # dump form: leading zero stripped with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - result = link_gnomad_variants_to_mapped_variants(session, [mocked_gnomad_variant_row], False) - assert result == 2 + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == {allele.id} session.commit() - gnomad_variant_comparator = TEST_GNOMAD_VARIANT.copy() - gnomad_variant_comparator.pop("creation_date") - gnomad_variant_comparator.pop("modification_date") - for mv in [mapped_variant1, mapped_variant2]: - assert len(mv.gnomad_variants) == 1 - for attr in gnomad_variant_comparator: - assert getattr(mv.gnomad_variants[0], attr) == gnomad_variant_comparator[attr] + assert len(_live_links_for(session, allele.id)) == 1 + - _verify_annotation_status(session, [mapped_variant1, mapped_variant2], TEST_GNOMAD_DATA_VERSION) +def test_returns_empty_set_when_no_alleles_match(session, mocked_gnomad_variant_row): + result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) + assert result == set() + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 + # No gnomAD variant is created when nothing matches the CAID. + assert len(session.scalars(select(GnomADVariant)).all()) == 0 diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index eac380862..d377f3fda 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1,9 +1,13 @@ import pytest +from sqlalchemy import select +from mavedb.models.allele import Allele from mavedb.models.enums.job_pipeline import DependencyType from mavedb.models.job_dependency import JobDependency from mavedb.models.job_run import JobRun from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -281,6 +285,87 @@ def setup_sample_variants_with_caid( return variant, mapped_variant +@pytest.fixture +def setup_sample_alleles_with_caid(session, with_populated_domain_data, sample_link_gnomad_variants_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the gnomAD linkage job. The allele carries the CAID matched by the mocked Athena row, and the + allele is the authoritative measurement for the variant so the bandaid seam writes its VAS row. + """ + score_set = session.get(ScoreSet, sample_link_gnomad_variants_run.job_params["score_set_id"]) + + variant = Variant( + urn="urn:variant:test-variant-with-allele-caid", + score_set_id=score_set.id, + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, + ) + allele = Allele( + vrs_digest="test-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NM_000000.1:c.1A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) + session.commit() + + mapping_record = MappingRecord( + variant_id=variant.id, + assay_level="genomic", + mapping_api_version="pytest.0.0", + ) + session.add(mapping_record) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_with_caid(session, setup_sample_alleles_with_caid): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying + the CAID the mocked gnomAD row matches. The authoritative allele is given a CAID with no gnomAD + match, so only the RT-derived allele can link. This isolates the requirement that gnomAD linkage + must cover the full allele set (authoritative + RT-derived), not just authoritative links — for + protein/coding score sets the genomic allele gnomAD knows is the RT-derived one. + """ + variant, authoritative_allele = setup_sample_alleles_with_caid + + # Authoritative allele's CAID intentionally has no gnomAD match, so it cannot be what links. + authoritative_allele.clingen_allele_id = "CA_NO_GNOMAD_MATCH" + session.add(authoritative_allele) + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-allele-vrs-digest", + level="genomic", + clingen_allele_id=VALID_CAID, + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000001.11:g.12345G>A", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele + + ## Uniprot Job Fixtures ## diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index fc8e211c0..23316adb8 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -4,12 +4,16 @@ pytest.importorskip("arq") -from unittest.mock import MagicMock, patch +from unittest.mock import patch +from sqlalchemy import select + +from mavedb.lib import gnomad as gnomad_lib +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -22,7 +26,7 @@ class TestLinkGnomadVariantsUnit: """Unit tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -30,7 +34,7 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test linking gnomAD variants when no mapped variants have CAIDs.""" + """No authoritative alleles with CAIDs -> the job succeeds with nothing to do.""" result = await link_gnomad_variants( mock_worker_ctx, 1, @@ -47,7 +51,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test linking gnomAD variants when no gnomAD variants match the CAIDs.""" @@ -55,7 +59,7 @@ async def test_link_gnomad_variants_no_gnomad_matches( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value={}, + return_value=[], ), patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -75,7 +79,7 @@ async def test_link_gnomad_variants_call_linking_method( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that the linking method is called when gnomAD variants match CAIDs.""" @@ -83,11 +87,11 @@ async def test_link_gnomad_variants_call_linking_method( with ( patch( "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", - return_value=[MagicMock()], + return_value=[object()], ), patch( - "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_mapped_variants", - return_value=1, + "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", + return_value=set(), ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -108,7 +112,7 @@ async def test_link_gnomad_variants_propagates_exceptions( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are propagated.""" @@ -134,7 +138,7 @@ async def test_link_gnomad_variants_propagates_exceptions( class TestLinkGnomadVariantsIntegration: """Integration tests for the link_gnomad_variants job.""" - async def test_link_gnomad_variants_no_variants_with_caids( + async def test_link_gnomad_variants_no_alleles_with_caids( self, session, with_populated_domain_data, @@ -142,17 +146,16 @@ async def test_link_gnomad_variants_no_variants_with_caids( mock_worker_ctx, sample_link_gnomad_variants_run, ): - """Test the end-to-end functionality of the link_gnomad_variants job when no variants have CAIDs.""" + """Test the end-to-end functionality of the link_gnomad_variants job when no alleles have CAIDs.""" result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify no annotations were rendered (since there were no variants with CAIDs) + # Verify no annotations were rendered (since there were no alleles with CAIDs) annotation_statuses = session.query(VariantAnnotationStatus).all() assert len(annotation_statuses) == 0 @@ -167,13 +170,13 @@ async def test_link_gnomad_variants_no_matching_caids( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job when no matching CAIDs are found.""" - # Update the created mapped variant to have a CAID that won't match any gnomAD data - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "NON_MATCHING_CAID" + # Update the allele to have a CAID that won't match any seeded gnomAD data + _, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "NON_MATCHING_CAID" session.commit() # Patch the athena engine to use the mock athena_engine fixture @@ -183,11 +186,10 @@ async def test_link_gnomad_variants_no_matching_caids( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there were variants with CAIDs) + # Verify a skipped annotation status was rendered (since there was an allele with a CAID) annotation_statuses = session.query(VariantAnnotationStatus).all() assert len(annotation_statuses) == 1 assert annotation_statuses[0].status == "skipped" @@ -204,10 +206,11 @@ async def test_link_gnomad_variants_successful_linking_independent( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking.""" + _, allele = setup_sample_alleles_with_caid # Patch the athena engine to use the mock athena_engine fixture with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -216,9 +219,15 @@ async def test_link_gnomad_variants_successful_linking_independent( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that a gnomAD variant was created and a live link to the allele established + assert len(session.scalars(select(GnomADVariant)).all()) > 0 + live_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(live_links) == 1 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -230,6 +239,147 @@ async def test_link_gnomad_variants_successful_linking_independent( session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_authoritative( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_rt_derived_allele_with_caid, + athena_engine, + ): + """gnomAD linkage must cover the full allele set, not just authoritative links: the + RT-derived allele carries the CAID gnomAD matches and must be linked. Per-variant VAS status, + however, is written only for the authoritative link (the interim bandaid) — so exactly one + annotation row is produced, keyed to the variant, never a second row for the RT-derived + allele.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + assert result.status == JobStatus.SUCCEEDED + + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_links = session.scalars( + select(GnomadAlleleLink).where( + GnomadAlleleLink.allele_id == rt_allele.id, + GnomadAlleleLink.current, + ) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no gnomAD match, so it gets no link. + assert ( + len( + session.scalars( + select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == authoritative_allele.id) + ).all() + ) + == 0 + ) + + # Annotation status is written only for the authoritative link: exactly one VAS row, for the + # variant. (Its status is "skipped" because the variant's authoritative allele had no match — + # cross-level resolution onto the RT-derived allele is deferred to the AnnotationEvent design.) + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].variant_id == variant.id + assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + + async def test_link_gnomad_variants_skips_allele_already_current( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """An allele already linked at the current gnomAD version is skipped: no Athena query, the + status reports SUCCESS/preexisting, and the existing link is not churned.""" + _, allele = setup_sample_alleles_with_caid + + # Simulate a prior run: a live link at the current gnomAD version. + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=1, + allele_number=2, + allele_frequency=0.5, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + with patch("mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids") as fetch_spy: + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + + # Link not churned: still exactly one, still live. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Status is SUCCESS, marked preexisting. + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + + async def test_link_gnomad_variants_force_refetches_without_churn( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + athena_engine, + ): + """force bypasses the skip and re-fetches, but the linker supersedes only on change, so a + forced re-run of unchanged data reports preexisting and does not churn the link.""" + _, allele = setup_sample_alleles_with_caid + + # Prior live link pointing at the same variant the Athena mock resolves to (1-12345-G-A). + gnomad_variant = GnomADVariant( + db_name="gnomAD", + db_identifier="1-12345-G-A", + db_version=GNOMAD_DATA_VERSION, + allele_count=23, + allele_number=32432423, + allele_frequency=23 / 32432423, + ) + session.add(gnomad_variant) + session.commit() + session.add(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)) + session.commit() + + sample_link_gnomad_variants_run.job_params = {**sample_link_gnomad_variants_run.job_params, "force": True} + session.commit() + + with ( + patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), + patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + side_effect=gnomad_lib.gnomad_variant_data_for_caids, + ) as fetch_spy, + ): + result = await link_gnomad_variants(mock_worker_ctx, sample_link_gnomad_variants_run.id) + + fetch_spy.assert_called_once() # force bypassed the version-keyed skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 + # Unchanged → no churn. + links = session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + async def test_link_gnomad_variants_successful_linking_pipeline( self, session, @@ -237,7 +387,7 @@ async def test_link_gnomad_variants_successful_linking_pipeline( mock_worker_ctx, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test the end-to-end functionality of the link_gnomad_variants job with successful linking in a pipeline.""" @@ -249,9 +399,8 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -274,7 +423,7 @@ async def test_link_gnomad_variants_exceptions_handled_by_decorators( with_gnomad_linking_job, mock_worker_ctx, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, athena_engine, ): """Test that exceptions during the linking process are handled by decorators.""" @@ -317,7 +466,7 @@ async def test_link_gnomad_variants_with_arq_context_independent( with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture.""" @@ -328,9 +477,8 @@ async def test_link_gnomad_variants_with_arq_context_independent( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -351,7 +499,7 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( athena_engine, sample_link_gnomad_variants_run_pipeline, sample_link_gnomad_variants_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that the link_gnomad_variants job works with the ARQ context fixture in a pipeline.""" @@ -362,9 +510,8 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( await arq_worker.async_run() await arq_worker.run_check() - # Verify that gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) > 0 + # Verify that allele links were created + assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -389,7 +536,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ with_gnomad_linking_job, athena_engine, sample_link_gnomad_variants_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -406,9 +553,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered annotation_statuses = session.query(VariantAnnotationStatus).all() @@ -427,7 +573,7 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline athena_engine, sample_link_gnomad_variants_pipeline, sample_link_gnomad_variants_run_pipeline, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): """Test that exceptions in the link_gnomad_variants job are handled with the ARQ context fixture.""" @@ -444,9 +590,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline await arq_worker.run_check() mock_send_slack_job_error.assert_called_once() - # Verify that no gnomAD variants were linked - gnomad_variants = session.query(GnomADVariant).all() - assert len(gnomad_variants) == 0 + # Verify that no allele links were created + assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered annotation_statuses = session.query(VariantAnnotationStatus).all() From c77c21b0fc35d3546b181441f3e6eaec26b18f26 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 14:34:01 -0700 Subject: [PATCH 26/93] feat: link VEP consequences to alleles with Ensembl-release versioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the VEP functional-consequence job (Step 2 of the annotation infrastructure migration) off MappedVariant onto the deduplicated allele model. - New ValidTime vep_allele_consequences table: a single allele-keyed row collapses record + link (the consequence is a scalar with no shared external entity, unlike gnomAD/ClinVar). One live consequence per allele via a partial unique index. - Job runs over the score set's full allele set (authoritative + RT-derived) via get_alleles_for_score_set + the shared grouping primitive; VAS still fans only to authoritative variants (the bandaid seam). - Version-key the refresh on the Ensembl release (/info/software): skip alleles already current, abort if the release can't be fetched. Supersede is value-keyed, not version-keyed — an unchanged consequence at a new release bumps source_version/access_date in place to avoid churning history. force bypasses the skip (e.g. after editing VEP_CONSEQUENCES). - A no-result is treated as a non-answer, never a negative: held consequences are not retired on an empty/failed VEP fetch. - Annotation links stay one-directional to Allele (no reverse back-ref). Adds lib + job tests covering linkage, RT-derived scope, version skip, in-place bump, supersede-on-change, no-result handling, and release-fetch failure. --- ...7c4e9d2f1b8_add_vep_allele_consequences.py | 65 ++ src/mavedb/lib/clingen/alleles.py | 17 +- src/mavedb/lib/vep.py | 108 ++- src/mavedb/models/__init__.py | 1 + src/mavedb/models/allele.py | 4 + src/mavedb/models/vep_allele_consequence.py | 75 ++ .../worker/jobs/external_services/vep.py | 594 +++++++-------- tests/lib/test_vep.py | 163 +++- tests/worker/jobs/conftest.py | 114 ++- .../external_services/network/test_vep.py | 37 +- .../worker/jobs/external_services/test_vep.py | 704 ++++++++++-------- 11 files changed, 1193 insertions(+), 689 deletions(-) create mode 100644 alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py create mode 100644 src/mavedb/models/vep_allele_consequence.py diff --git a/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py new file mode 100644 index 000000000..8b655f625 --- /dev/null +++ b/alembic/versions/a7c4e9d2f1b8_add_vep_allele_consequences.py @@ -0,0 +1,65 @@ +"""add vep_allele_consequences table + +Revision ID: a7c4e9d2f1b8 +Revises: e5f7a9c1b3d4 +Create Date: 2026-06-22 + +New valid-time table holding a deduplicated allele's VEP functional consequence, replacing the frozen +vep_functional_consequence/vep_access_date columns on mapped_variants for new-model writes (Step 2 of +the annotation infrastructure migration, docs/design/annotation-infrastructure-migration.md). A row is +live while valid_to is NULL; the partial unique index enforces a single live consequence per allele +(VEP's most-severe consequence is one current value, so a change supersedes rather than accumulates). +functional_consequence is nullable (reserved for a future negative cache). source_version is the +Ensembl release the consequence was resolved under (coordinated software + transcript set + vocabulary), +which version-keys the refresh skip like gnomAD's db_version; access_date is retained as a "last +confirmed" audit stamp. The VEP columns on mapped_variants are left untouched (frozen serving). +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7c4e9d2f1b8" +down_revision = "e5f7a9c1b3d4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "vep_allele_consequences", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("functional_consequence", sa.String(), nullable=True), + sa.Column("source_version", sa.String(), nullable=False), + sa.Column("access_date", sa.Date(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_vep_allele_consequences_allele_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_vep_allele_consequences_allele_id", + "vep_allele_consequences", + ["allele_id"], + ) + # One live consequence per allele: VEP's most-severe consequence is a single current value, so a + # changed result supersedes the prior row rather than accumulating one live row per access. + op.create_index( + "uq_vep_allele_consequences_live", + "vep_allele_consequences", + ["allele_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_vep_allele_consequences_live", table_name="vep_allele_consequences") + op.drop_index("ix_vep_allele_consequences_allele_id", table_name="vep_allele_consequences") + op.drop_table("vep_allele_consequences") diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 79baf47ff..a2a962765 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -25,6 +25,10 @@ class ScoreSetAlleleRow(NamedTuple): ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be the authoritative measurement for one variant and an RT-derived equivalence for another. + + ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the + VEP job can build its HGVS payload without a second query. They are optional with a ``None`` + default so payloads keying only on the CAID (gnomAD/ClinVar) need not name them. """ allele_id: int @@ -32,6 +36,9 @@ class ScoreSetAlleleRow(NamedTuple): clingen_allele_id: str | None variant_id: int is_authoritative: bool + hgvs_g: str | None = None + hgvs_c: str | None = None + hgvs_p: str | None = None def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: @@ -51,6 +58,9 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.clingen_allele_id, Variant.id.label("variant_id"), MappingRecordAllele.is_authoritative, + Allele.hgvs_g, + Allele.hgvs_c, + Allele.hgvs_p, ) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) @@ -61,7 +71,12 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl .where(Allele.post_mapped.is_not(None)) ).all() - return [ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative) for r in rows] + return [ + ScoreSetAlleleRow( + r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative, r.hgvs_g, r.hgvs_c, r.hgvs_p + ) + for r in rows + ] @dataclass diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index a7d4e7b3c..4b8ac0342 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -4,11 +4,16 @@ import functools import logging import os -from typing import Optional, Sequence +from datetime import date +from typing import Mapping, Optional, Sequence import requests +from sqlalchemy import select +from sqlalchemy.orm import Session +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.utils import request_with_backoff +from mavedb.models.vep_allele_consequence import VepAlleleConsequence logger = logging.getLogger(__name__) @@ -185,3 +190,104 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O result[hgvs] = most_severe_consequence return result + + +async def get_ensembl_release() -> str: + """Return the current Ensembl release the REST API is serving, e.g. ``"116"`` (``/info/software``). + + An Ensembl release is coordinated — software, transcript set, and consequence vocabulary all bump + together under one number — so this single value version-keys VEP results the way gnomAD keys on its + data version. The job stamps it on each consequence and skips re-querying alleles already live at the + current release. Raises on failure: the version is load-bearing for the skip, so a job that cannot + determine it must abort rather than mis-version its writes. + """ + headers = {"Content-Type": "application/json", "Accept": "application/json"} + loop = asyncio.get_running_loop() + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="GET", + url=f"{ENSEMBL_API_URL}/info/software", + headers=headers, + timeout=30, + ), + ) + return str(response.json()["release"]) + + +def link_vep_consequences_to_alleles( + db: Session, + consequence_by_allele_id: Mapping[int, Optional[str]], + *, + source_version: str, + access_date: date, +) -> set[int]: + """Store VEP consequences against deduplicated alleles, superseding only on change. + + ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run + (``None`` when VEP + Variant Recoder found nothing). ``source_version`` is the Ensembl release the + run resolved against. Each allele holds at most one live :class:`VepAlleleConsequence`, handled per + allele: + + - **unchanged** (live row already carries this consequence): advance ``source_version`` and + ``access_date`` in place — no supersede. Supersede is value-keyed, not version-keyed: a new + release that resolves the same categorical consequence must not fabricate a transaction-time + boundary, which would churn history every release. + - **new or changed** (no live row, or a different consequence): supersede keyed on ``allele_id`` + (retire the old, insert the successor stamped with this ``source_version``/``access_date``). + - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. + Log a warning if VEP found no consequence for an allele which previously had a live consequence. + + Does not commit. Returns the allele ids whose consequence was created or superseded this run. + """ + save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) + logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) + + changed_allele_ids: set[int] = set() + for allele_id, consequence in consequence_by_allele_id.items(): + live = db.scalar( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ) + + # VEP found nothing this run. Do not overwrite a held consequence with a null result. + if consequence is None: + if live is not None: + logger.warning( + f"VEP found no consequence for allele {allele_id} this run; leaving prior consequence " + f"'{live.functional_consequence}' in place.", + extra=logging_context(), + ) + + continue + + # Unchanged: advance version/freshness in place. + if live is not None and live.functional_consequence == consequence: + live.source_version = source_version + live.access_date = access_date + continue + + # New or changed consequence: retire any live row, insert the successor. + VepAlleleConsequence.supersede_live_where( + db, + [ + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + ) + ], + VepAlleleConsequence.allele_id == allele_id, + ) + changed_allele_ids.add(allele_id) + + save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + logger.info( + msg=f"Created or superseded {len(changed_allele_ids)} VEP allele consequences this run.", + extra=logging_context(), + ) + return changed_allele_ids diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index 5284da11b..b18e20ea7 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -43,4 +43,5 @@ "variant_annotation_status", "variant", "variant_translation", + "vep_allele_consequence", ] diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index 54ce07e50..a57c4c802 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -51,6 +51,10 @@ def _transcript_expression(cls): back_populates="allele", ) + # Annotation links (VEP, gnomAD, ClinVar) deliberately carry no reverse collection here — they are + # one-directional annotation->Allele, navigated set-wise from the link tables, not from an Allele + # instance. Keep new annotation links one-directional unless a read path needs the navigation. + __table_args__ = ( UniqueConstraint("vrs_digest", name="uq_alleles_vrs_digest"), Index("ix_alleles_vrs_digest", "vrs_digest"), diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py new file mode 100644 index 000000000..de41230a5 --- /dev/null +++ b/src/mavedb/models/vep_allele_consequence.py @@ -0,0 +1,75 @@ +from datetime import date +from typing import TYPE_CHECKING + +from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + + +class VepAlleleConsequence(ValidTime, Base): + """Valid-time VEP functional-consequence result for a deduplicated :class:`Allele`. + + Replaces the frozen ``vep_functional_consequence``/``vep_access_date`` columns on + ``MappedVariant`` for new-model writes (Step 2 of the annotation infrastructure migration, + docs/design/annotation-infrastructure-migration.md). A row is live while ``valid_to`` is NULL; + the partial unique index enforces **a single live consequence per allele** — VEP's most-severe + consequence is one current value, so a changed result supersedes the prior row rather than + accumulating. This matches the gnomAD link shape, not ClinVar's multi-live shape. + + ``source_version`` is the Ensembl release the consequence was resolved under (e.g. ``"116"``, + from ``/info/software``). An Ensembl release is coordinated — software + transcript set + + consequence vocabulary all bump together under one number — so this single value version-keys the + upstream result exactly like gnomAD's ``db_version``. The job skips re-querying any allele already + live at the current release. What it does **not** capture is our own ``VEP_CONSEQUENCES`` severity + ordering (the list we pick "most severe" from); a change to that is a manual ``force`` re-run, not + an automatic supersede. + + Supersede is deliberately **value-keyed, not version-keyed** (the one divergence from gnomAD): a VEP + consequence is categorical and usually identical across releases, so superseding on every release + bump would churn history every quarter with rows recording "still missense, still missense." Instead + a new release that resolves the *same* consequence advances ``source_version``/``access_date`` in + place — no supersede — and only a *changed* consequence retires the old row and inserts a successor. + The trade-off: the live row's ``source_version`` is the latest release that confirmed the value, not + the release it first appeared; acceptable because it describes the currently-held value's + provenance, not when it became true. ``access_date`` is retained as a human-facing "last confirmed" + audit stamp; it is no longer load-bearing for the skip. + + ``functional_consequence`` is nullable to leave room for a future negative cache (NULL = "VEP ran + and found nothing"); the current job writes only non-null consequences and re-queries no-result + alleles each run, mirroring gnomAD's no-match handling. + """ + + __tablename__ = "vep_allele_consequences" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + functional_consequence = Column(String, nullable=True) + source_version: Mapped[str] = Column(String, nullable=False) + access_date: Mapped[date] = Column(Date, nullable=False) + + allele: Mapped["Allele"] = relationship("Allele") + + __table_args__ = ( + Index( + "ix_vep_allele_consequences_allele_id", + "allele_id", + ), + # At most one live consequence per allele. A changed result supersedes (retires the old, + # inserts the new) rather than accumulating; superseded rows stay for point-in-time queries. + # Only the live row participates in this constraint. + Index( + "uq_vep_allele_consequences_live", + "allele_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index ba577f230..02f6a4498 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -1,10 +1,8 @@ """VEP functional consequence jobs for variant effect prediction. -This module handles the submission and processing of variant effect predictions -using the Ensembl VEP API. - -The processing is asynchronous, requiring batch submission of HGVS strings -to the VEP API with fallback to Variant Recoder when necessary. +This module links deduplicated alleles to their Ensembl VEP functional consequence. Submission is +batched against the VEP API, with a Variant Recoder fallback for HGVS strings VEP cannot resolve +directly (notably protein HGVS). """ import asyncio @@ -15,14 +13,20 @@ from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.lib.utils import batched -from mavedb.lib.vep import VEP_CONSEQUENCES, get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + VEP_CONSEQUENCES, + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -34,17 +38,160 @@ _RECODER_CONCURRENCY = int(os.getenv("RECODER_CONCURRENCY", "5")) +def _annotate_vep( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a VEP_FUNCTIONAL_CONSEQUENCE annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for VEP's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance (which variant's mapping drove the linkage). See + docs/design/allele-annotation-status.md. + + VEP carries no version string, so the VAS row is written with ``version=None`` (unlike gnomAD, + which keys on its data version). + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + ) + + +def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: + """Build VEP's HGVS input for an allele: each allele will only have one of these, so the first + non-null is the one to submit. + """ + return row.hgvs_g or row.hgvs_c or row.hgvs_p or None + + +async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> dict[str, str]: + """Resolve a set of HGVS strings to their most-severe VEP consequence. + + Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry + with a null consequence is treated as a miss — VEP knew the variant but could not classify it). + Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to + the original HGVS. Returns only the HGVS strings that resolved to a consequence; absent keys are + failures the caller treats as no-result. + """ + all_consequences: dict[str, str] = {} + batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) + + # --- Phase 1: initial VEP pass --- + all_missing_hgvs: set[str] = set() + for batch_idx, batch in enumerate(batches): + consequences = await get_functional_consequence(list(batch)) + hit_consequences = {h: c for h, c in consequences.items() if c is not None} + all_consequences.update(hit_consequences) + all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) + + job_manager.update_progress( + int((batch_idx + 1) / len(batches) * 33), + 100, + f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + ) + + logger.info( + msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} HGVS strings require Variant Recoder fallback.", + extra=job_manager.logging_context(), + ) + + if not all_missing_hgvs: + return all_consequences + + # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- + recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) + semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) + completed_recoder_batches = 0 + + async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, list[str]]: + nonlocal completed_recoder_batches + async with semaphore: + result = await run_variant_recoder(batch) + completed_recoder_batches += 1 + job_manager.update_progress( + 33 + int(completed_recoder_batches / total * 33), + 100, + f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", + ) + return result + + total_recoder_batches = len(recoder_batch_list) + recoder_results = await asyncio.gather( + *[_recoder_with_semaphore(list(b), total_recoder_batches) for b in recoder_batch_list], + return_exceptions=True, + ) + + first_exception = next((r for r in recoder_results if isinstance(r, BaseException)), None) + if first_exception is not None: + successful_batches = sum(1 for r in recoder_results if not isinstance(r, BaseException)) + logger.error( + msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {first_exception}", + extra=job_manager.logging_context(), + ) + raise first_exception + + hgvs_to_genomic: dict[str, list[str]] = {} + for result in recoder_results: + hgvs_to_genomic.update(result) # type: ignore[arg-type] + + logger.info( + msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", + extra=job_manager.logging_context(), + ) + + # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- + all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) + recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) + all_recoded_consequences: dict[str, str | None] = {} + + for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): + all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + job_manager.update_progress( + 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), + 100, + f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", + ) + + # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. + for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): + recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] + most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) + if most_severe: + all_consequences[original_hgvs] = most_severe + + return all_consequences + + @with_pipeline_management async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate VEP functional consequence predictions for all mapped variants in a ScoreSet. + """Link deduplicated alleles to their VEP functional consequence. - This function retrieves all mapped variants with a populated hgvs_assay_level field for a given - ScoreSet and submits them to the Ensembl VEP API in configurable batches. It handles fallback - to the Variant Recoder API for variants that cannot be processed by VEP directly. + Runs over the score set's current alleles (authoritative and RT-derived), submits each allele's + HGVS to VEP (with Variant Recoder fallback), and stores the most-severe consequence in a valid-time + :class:`VepAlleleConsequence`, superseding only on change. Job Parameters: - - score_set_id (int): The ID of the ScoreSet containing mapped variants. + - score_set_id (int): The ID of the ScoreSet whose alleles to process. - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the current-release skip and re-query every HGVS-bearing + allele. The linker still supersedes only on a value change, so a forced re-run of unchanged + data writes no new rows. Use for re-ingestion, to heal suspected corruption, or after editing + the VEP_CONSEQUENCES severity ordering (a change the release version cannot see). Args: ctx (dict): The job context dictionary. @@ -52,7 +199,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager (JobManager): Manager for job lifecycle and DB operations. Returns: - JobExecutionOutcome: Outcome with counts of processed, successful, and failed variants. + JobExecutionOutcome: outcome with per-allele created/preexisting/skipped counts. """ job = job_manager.get_job() @@ -62,6 +209,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan # Safely ignore mypy warnings here, as params were checked above. score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] job_manager.save_to_context( { @@ -71,348 +219,136 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan "correlation_id": correlation_id, } ) - job_manager.update_progress(0, 100, "Starting VEP population.") - logger.info(msg="Started VEP population", extra=job_manager.logging_context()) - - mapped_variants = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), - MappedVariant.post_mapped.isnot(None), - ) - ).all() + job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") + logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) + + # One work-unit per allele (payload = HGVS; alleles without one are skipped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele VEP is most reliable on + # is often the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=_vep_hgvs_payload, + ) - if not mapped_variants: + num_alleles_with_hgvs = len(allele_data) + job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) + + if not allele_data: logger.warning( - msg=f"No mapped variants found for score set {score_set.urn}. Skipped VEP population.", + msg="No current alleles with HGVS were found for this score set. Skipping VEP linkage (nothing to do).", extra=job_manager.logging_context(), ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "variants_processed": 0, - "variants_with_consequences": 0, - "variants_without_consequences": 0, - "variants_recoder_failed": 0, - } - ) - - job_manager.save_to_context({"total_variants_to_process": len(mapped_variants)}) - logger.info( - msg=f"Found {len(mapped_variants)} mapped variants for VEP processing", - extra=job_manager.logging_context(), - ) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - mapped_variants_by_id = {mv.id: mv for mv in mapped_variants} - - # Extract HGVS strings; skip and annotate variants that have none. - hgvs_and_mapped_variant_id_pairs: list[tuple[str, int]] = [] - - for mapped_variant in mapped_variants: - if not mapped_variant.hgvs_assay_level: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={"error_message": "Mapped variant does not have an associated HGVS string."}, - ) - logger.debug("Mapped variant does not have an associated HGVS string.", extra=job_manager.logging_context()) - continue - - hgvs_and_mapped_variant_id_pairs.append((mapped_variant.hgvs_assay_level, mapped_variant.id)) # type: ignore - - batches = list(batched(hgvs_and_mapped_variant_id_pairs, _VEP_BATCH_SIZE)) - - job_manager.save_to_context({"vep_batches": len(batches)}) - logger.debug( - msg=f"Prepared {len(batches)} VEP batches ({_VEP_BATCH_SIZE} variants/batch)", - extra=job_manager.logging_context(), - ) - - # --- Phase 1: Initial VEP pass --- - all_consequences: dict[str, str] = {} - all_missing_hgvs: set[str] = set() - - for batch_idx, batch in enumerate(batches): - logger.debug( - msg=f"Processing VEP batch {batch_idx + 1}/{len(batches)}", - extra=job_manager.logging_context(), - ) - - hgvs_strings, mapped_variant_ids = map(list, zip(*batch)) # type: ignore - - consequences = await get_functional_consequence(hgvs_strings) - logger.debug( - msg=f"Received consequences for {len(consequences)} variants in VEP batch {batch_idx + 1}", - extra=job_manager.logging_context(), + data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} ) - # Only store variants where VEP returned an actual consequence string. A None value - # means VEP knew the variant but couldn't classify it — treat that the same as absent - # and route to Recoder so we have the best chance of getting a consequence. - hit_consequences = {h: c for h, c in consequences.items() if c is not None} - all_consequences.update(hit_consequences) - - missing_hgvs = set(hgvs_strings) - set(hit_consequences.keys()) - for hgvs, mapped_variant_id in zip(hgvs_strings, mapped_variant_ids): - if hgvs in missing_hgvs: - all_missing_hgvs.add(hgvs) - - progress_pct = int((batch_idx + 1) / len(batches) * 33) - job_manager.save_to_context( - { - "initial_vep_batches_processed": batch_idx + 1, - "missing_hgvs_count": len(all_missing_hgvs), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed initial VEP batch {batch_idx + 1}/{len(batches)}", + all_allele_ids = set(allele_data.keys()) + + # The Ensembl release version-keys the run (coordinated software + transcript set + vocabulary). It + # is load-bearing for the skip below, so a failure here aborts the job rather than mis-versioning + # writes (the exception propagates to the job decorators). + ensembl_release = await get_ensembl_release() + job_manager.save_to_context({"ensembl_release": ensembl_release}) + + def alleles_at_current_release(allele_ids: set[int]) -> set[int]: + """Allele ids (within the given set) holding a live VEP consequence at the current Ensembl release.""" + if not allele_ids: + return set() + return set( + job_manager.db.scalars( + select(VepAlleleConsequence.allele_id) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + .where(VepAlleleConsequence.source_version == ensembl_release) + ).all() ) - logger.info( - msg=f"Completed initial VEP processing. {len(all_missing_hgvs)} variants require Variant Recoder fallback.", - extra=job_manager.logging_context(), + # Cost: skip alleles already resolved at the current Ensembl release (they cannot change without a + # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES + # severity ordering we have since edited; the linker still supersedes only on a value change, so a + # forced no-op writes nothing. + already_current = set() if force else alleles_at_current_release(all_allele_ids) + hgvs_by_allele = {aid: allele_data[aid].payload for aid in allele_data if aid not in already_current} + unique_hgvs = sorted(set(hgvs_by_allele.values())) + job_manager.save_to_context( + { + "num_alleles_already_current": len(already_current), + "num_hgvs_to_query": len(unique_hgvs), + "force": force, + } ) - # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- - hgvs_to_genomic: dict[str, list[str]] = {} - recoder_missing_hgvs: set[str] = set() - - if all_missing_hgvs: - logger.info( - msg=f"Running Variant Recoder for {len(all_missing_hgvs)} HGVS strings", - extra=job_manager.logging_context(), - ) - - recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) - - logger.debug( - msg=f"Running {len(recoder_batch_list)} Variant Recoder batches with concurrency {_RECODER_CONCURRENCY}", - extra=job_manager.logging_context(), - ) - - semaphore = asyncio.Semaphore(_RECODER_CONCURRENCY) - completed_recoder_batches = 0 - - async def _recoder_with_semaphore(batch: list[str], batch_idx: int, total: int) -> dict[str, list[str]]: - nonlocal completed_recoder_batches - async with semaphore: - logger.debug( - msg=f"Starting Variant Recoder batch {batch_idx + 1}/{total} ({len(batch)} HGVS strings)", - extra=job_manager.logging_context(), - ) - result = await run_variant_recoder(batch) - completed_recoder_batches += 1 - logger.debug( - msg=f"Completed Variant Recoder batch {completed_recoder_batches}/{total} ({len(result)} variants recoded)", - extra=job_manager.logging_context(), - ) - progress_pct = 33 + int(completed_recoder_batches / total * 33) - job_manager.update_progress( - progress_pct, - 100, - f"Completed Variant Recoder batch {completed_recoder_batches}/{total}", - ) - return result - - total_recoder_batches = len(recoder_batch_list) - recoder_results = await asyncio.gather( - *[ - _recoder_with_semaphore(list(recoder_batch), idx, total_recoder_batches) - for idx, recoder_batch in enumerate(recoder_batch_list) - ], - return_exceptions=True, - ) - - successful_batches = sum(1 for r in recoder_results if not isinstance(r, Exception)) - - first_exception = next((r for r in recoder_results if isinstance(r, Exception)), None) - if first_exception is not None: - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {str(first_exception)}", - extra=job_manager.logging_context(), - ) - raise first_exception - - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] - - job_manager.save_to_context( - { - "variant_recoder_batches_processed": len(recoder_batch_list), - "recoded_variants_count": len(hgvs_to_genomic), - } - ) - logger.info( - msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} variants successfully recoded.", - extra=job_manager.logging_context(), + changed_allele_ids: set[int] = set() + if unique_hgvs: + job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") + consequences_by_hgvs = await _resolve_consequences(unique_hgvs, job_manager) + consequence_by_allele_id = {aid: consequences_by_hgvs.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} + changed_allele_ids = link_vep_consequences_to_alleles( + job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() ) - - # --- Phase 3: VEP pass on the recoded genomic HGVS strings --- - # hgvs_to_genomic maps original HGVS → list[str]; flatten to a deduplicated list of - # genomic strings before batching with VEP. - all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) - recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) - all_recoded_consequences: dict[str, str | None] = {} - - for recoded_vep_batch_idx, recoded_vep_batch in enumerate(recoded_vep_batch_list): - logger.debug( - msg=f"Processing recoded HGVS VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - extra=job_manager.logging_context(), - ) - - recoded_vep_consequences = await get_functional_consequence(recoded_vep_batch) - all_recoded_consequences.update(recoded_vep_consequences) - - progress_pct = 66 + int((recoded_vep_batch_idx + 1) / len(recoded_vep_batch_list) * 33) - job_manager.save_to_context( - { - "recoded_vep_batches_processed": recoded_vep_batch_idx + 1, - "recoded_consequences_count": len(all_recoded_consequences), - } - ) - job_manager.update_progress( - progress_pct, - 100, - f"Processed recoded VEP batch {recoded_vep_batch_idx + 1}/{len(recoded_vep_batch_list)}", - ) - + job_manager.db.flush() + else: logger.info( - msg=f"Completed recoded VEP processing. {len(all_recoded_consequences)} recoded consequences retrieved.", + msg="All HGVS-bearing alleles are already resolved at the current Ensembl release; skipping VEP query.", extra=job_manager.logging_context(), ) + job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") + + # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting + # if already current or re-confirmed unchanged, SKIPPED if no live consequence (VEP had no result). + live_now = job_manager.db.execute( + select(VepAlleleConsequence.allele_id, VepAlleleConsequence.functional_consequence) + .where(VepAlleleConsequence.allele_id.in_(all_allele_ids)) + .where(VepAlleleConsequence.current) + .where(VepAlleleConsequence.functional_consequence.isnot(None)) + ).all() + consequence_now = {r.allele_id: r.functional_consequence for r in live_now} - # Map most-severe consequence from recoded genomic HGVS back to the original HGVS. - for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): - recoded_consequences_for_variant = [ - c for recoded_hgvs in recoded_hgvs_list if (c := all_recoded_consequences.get(recoded_hgvs)) - ] - - if recoded_consequences_for_variant: - most_severe = next( - (c for c in VEP_CONSEQUENCES if c in recoded_consequences_for_variant), - None, - ) - if most_severe: - all_consequences[original_hgvs] = most_severe - logger.debug( - msg=f"Selected most severe consequence '{most_severe}' for {original_hgvs}", - extra=job_manager.logging_context(), - ) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + created_count = preexisting_count = skipped_count = 0 + for allele_id, entry in allele_data.items(): + if allele_id in consequence_now: + action = "created" if allele_id in changed_allele_ids else "preexisting" + if action == "created": + created_count += 1 else: - logger.debug( - msg=f"Could not retrieve functional consequences for any recoded variants of {original_hgvs}", - extra=job_manager.logging_context(), - ) - - recoder_missing_hgvs = all_missing_hgvs - set(hgvs_to_genomic.keys()) - - # --- Phase 4: Annotate outcomes and update mapped variants in a single pass --- - - # HGVS strings that went through both VEP passes but still have no consequence. - all_processed_hgvs = {h for h, _ in hgvs_and_mapped_variant_id_pairs} - vep_failed_hgvs = all_processed_hgvs - set(all_consequences.keys()) - recoder_missing_hgvs - - variants_processed = 0 - variants_with_consequences = 0 - variants_without_consequences = 0 - variants_recoder_failed = 0 - - for hgvs_string, mapped_variant_id in hgvs_and_mapped_variant_id_pairs: - mapped_variant = mapped_variants_by_id.get(mapped_variant_id) # type: ignore - if mapped_variant is None: - continue - - consequence = all_consequences.get(hgvs_string) - if consequence: - mapped_variant.vep_functional_consequence = consequence - mapped_variant.vep_access_date = date.today() - job_manager.db.add(mapped_variant) - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": {"functional_consequence": consequence}}, - ) - variants_with_consequences += 1 - logger.debug( - msg=f"Set consequence '{consequence}' for mapped variant {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in vep_failed_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "VEP could not determine a functional consequence for this variant, even after Variant Recoder fallback.", + preexisting_count += 1 + _annotate_vep( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SUCCESS, + metadata={ + "functional_consequence": consequence_now[allele_id], + "hgvs": entry.payload, + "action": action, }, ) - variants_without_consequences += 1 - logger.debug( - msg=f"Recorded VEP failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) - elif hgvs_string in recoder_missing_hgvs: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "Variant Recoder could not recode this HGVS string to a genomic equivalent.", - }, - ) - variants_recoder_failed += 1 - logger.debug( - msg=f"Recorded Variant Recoder failure for mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string})", - extra=job_manager.logging_context(), - ) else: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "Variant was not classified by any VEP outcome branch. This is a bug.", - }, - ) - variants_without_consequences += 1 - logger.warning( - msg=f"Unexpected state: mapped_variant_id {mapped_variant_id} (HGVS: {hgvs_string}) was not classified by any outcome branch.", - extra=job_manager.logging_context(), + skipped_count += 1 + _annotate_vep( + annotation_manager, + entry.authoritative_variant_ids, + AnnotationStatus.SKIPPED, + failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + error_message="VEP could not determine a functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": entry.payload}, ) - variants_processed += 1 - annotation_manager.flush() - job_manager.db.flush() + outcome_data = { + "created_allele_count": created_count, + "preexisting_allele_count": preexisting_count, + "skipped_allele_count": skipped_count, + } + job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP functional consequence prediction for {variants_with_consequences}/{variants_processed} variants.", + f"Completed VEP linkage: {created_count} created, {preexisting_count} preexisting, {skipped_count} skipped.", ) - logger.info( - msg=f"Completed VEP prediction: {variants_with_consequences} with consequences, {variants_without_consequences} without, {variants_recoder_failed} recoder failed", - extra=job_manager.logging_context(), - ) - + logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "variants_processed": variants_processed, - "variants_with_consequences": variants_with_consequences, - "variants_without_consequences": variants_without_consequences, - "variants_recoder_failed": variants_recoder_failed, - } - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 2e9e1faef..0d9473f1b 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -4,11 +4,20 @@ logic correctly handles the actual Ensembl REST API response shapes. """ +from datetime import date, timedelta from unittest.mock import MagicMock, patch import pytest +from sqlalchemy import select -from mavedb.lib.vep import get_functional_consequence, run_variant_recoder +from mavedb.lib.vep import ( + get_ensembl_release, + get_functional_consequence, + link_vep_consequences_to_alleles, + run_variant_recoder, +) +from mavedb.models.allele import Allele +from mavedb.models.vep_allele_consequence import VepAlleleConsequence def _mock_response(data) -> MagicMock: @@ -240,3 +249,155 @@ async def test_raises_if_more_than_200_variants(self): """Passing more than 200 HGVS strings raises ValueError before any HTTP call.""" with pytest.raises(ValueError, match="maximum of 200"): await get_functional_consequence(["NM_007294.4:c.1A>T"] * 201) + + +### Tests for get_ensembl_release function ### + + +@pytest.mark.asyncio +async def test_get_ensembl_release_returns_release_as_string(): + """The /info/software release integer is returned as a string for use as source_version.""" + with patch("mavedb.lib.vep.request_with_backoff", return_value=_mock_response({"release": 116})): + assert await get_ensembl_release() == "116" + + +### Tests for link_vep_consequences_to_alleles function ### + + +def _make_allele(session, *, vrs_digest, level="genomic"): + """Create and persist a deduplicated Allele.""" + allele = Allele(vrs_digest=vrs_digest, level=level) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _live_rows_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + + +def _all_rows_for(session, allele_id): + return session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele_id)).all() + + +def test_link_vep_creates_new_consequence(session): + """A consequence for an allele with no live row creates a single live row and is reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == {allele.id} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + assert live[0].access_date == date.today() + + +def test_link_vep_unchanged_bumps_version_and_date_in_place(session): + """Re-confirming an unchanged consequence at a new release advances source_version and access_date + in place — no supersede, no new valid-time boundary, and the allele is not reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + # One row, still live, never retired — version and access_date advanced in place. + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 1 + assert all_rows[0].valid_to is None + assert all_rows[0].source_version == "116" + assert all_rows[0].access_date == date.today() + + +def test_link_vep_changed_consequence_supersedes(session): + """A changed consequence retires the live row and inserts the successor — exactly one live row, + keyed on allele_id, with the old one preserved as retired history.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == {allele.id} + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == "116" + + all_rows = _all_rows_for(session, allele.id) + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + +def test_link_vep_none_leaves_live_row_untouched(session): + """A transient None result must not overwrite a held consequence: the live row is left intact + (value, version, and date) and the allele is not reported changed.""" + allele = _make_allele(session, vrs_digest="vrs-1") + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), + ) + ) + session.commit() + + changed = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + live = _live_rows_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + # Not re-confirmed -> neither version nor access_date advanced. + assert live[0].source_version == "115" + assert live[0].access_date == date.today() - timedelta(days=90) + + +def test_link_vep_none_with_no_live_row_writes_nothing(session): + """A None result for an allele with no live row writes nothing (the allele is re-queried next run), + mirroring gnomAD's no-match handling.""" + allele = _make_allele(session, vrs_digest="vrs-1") + + changed = link_vep_consequences_to_alleles( + session, {allele.id: None}, source_version="116", access_date=date.today() + ) + session.commit() + + assert changed == set() + assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index d377f3fda..6b7309fd2 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1340,8 +1340,14 @@ def sample_populate_vep_run_pipeline( @pytest.fixture -def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a variant and mapped variant with hgvs_assay_level for VEP testing.""" +def setup_sample_alleles_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up new-model rows (Variant + live MappingRecord + authoritative MappingRecordAllele + Allele) + for the VEP consequence job. The allele carries an HGVS the job submits to VEP and is the + authoritative measurement for the variant, so the bandaid seam writes its per-variant VAS row. + + The HGVS lives on ``Allele.hgvs_c`` (a coding HGVS VEP resolves directly) — the new job reads its + submission string from the allele, not from ``MappedVariant.hgvs_assay_level``. + """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) variant = Variant( @@ -1351,28 +1357,42 @@ def setup_sample_variants_for_vep(session, with_populated_domain_data, mock_work hgvs_pro="NP_009225.1:p.Cys2Tyr", data={"hgvs_c": "NM_007294.4:c.5A>G", "hgvs_p": "NP_009225.1:p.Cys2Tyr"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-allele-vrs-digest", + level="cdna", + hgvs_c="NM_007294.4:c.5A>G", + post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NM_007294.4:c.5A>G", "syntax": "hgvs.c"}]}, - hgvs_assay_level="NM_007294.4:c.5A>G", + assay_level="cdna", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele @pytest.fixture -def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): - """Setup a protein HGVS variant (NP_ accession) that VEP cannot resolve directly. +def setup_sample_protein_allele_for_vep(session, with_populated_domain_data, mock_worker_ctx, sample_populate_vep_run): + """Set up an allele whose only HGVS is a protein HGVS (NP_ accession) that VEP cannot resolve + directly. - VEP's /vep/human/hgvs endpoint does not return results for protein HGVS strings like - NP_009225.1:p.Val1696His, so these must be recoded via Variant Recoder first. This fixture - exercises the recoder fallback path end-to-end. + VEP's /vep/human/hgvs endpoint returns no consequence for protein HGVS like + NP_009225.1:p.Val1696His, so the job must fall back to Variant Recoder. ``hgvs_g``/``hgvs_c`` are + left unset so the VEP payload resolves to ``hgvs_p``, exercising the recoder fallback path. """ score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) @@ -1382,17 +1402,61 @@ def setup_sample_protein_variant_for_vep(session, with_populated_domain_data, mo hgvs_pro="NP_009225.1:p.Val1696His", data={"hgvs_p": "NP_009225.1:p.Val1696His"}, ) - session.add(variant) + allele = Allele( + vrs_digest="test-vep-protein-allele-vrs-digest", + level="protein", + hgvs_p="NP_009225.1:p.Val1696His", + post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, + ) + session.add_all([variant, allele]) session.commit() - mapped_variant = MappedVariant( + mapping_record = MappingRecord( variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": "NP_009225.1:p.Val1696His", "syntax": "hgvs.p"}]}, - hgvs_assay_level="NP_009225.1:p.Val1696His", + assay_level="protein", + mapping_api_version="pytest.0.0", ) - session.add(mapped_variant) + session.add(mapping_record) session.commit() - return variant, mapped_variant + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=allele.id, + is_authoritative=True, + ) + ) + session.commit() + return variant, allele + + +@pytest.fixture +def setup_rt_derived_allele_for_vep(session, setup_sample_alleles_for_vep): + """Add a NON-authoritative (RT-derived) allele to the variant's current mapping record, carrying a + genomic HGVS of its own. Isolates the requirement that VEP linkage covers the full allele set + (authoritative + RT-derived), while the per-variant VAS fan-out stays authoritative-only. + """ + variant, authoritative_allele = setup_sample_alleles_for_vep + + mapping_record = session.scalars( + select(MappingRecord).where(MappingRecord.variant_id == variant.id, MappingRecord.current) + ).one() + + rt_allele = Allele( + vrs_digest="test-rt-derived-vep-allele-vrs-digest", + level="genomic", + hgvs_g="NC_000017.11:g.43124027T>C", + post_mapped={"type": "Allele", "expressions": [{"value": "NC_000017.11:g.43124027T>C", "syntax": "hgvs.g"}]}, + ) + session.add(rt_allele) + session.commit() + + session.add( + MappingRecordAllele( + mapping_record_id=mapping_record.id, + allele_id=rt_allele.id, + is_authoritative=False, + ) + ) + session.commit() + return variant, authoritative_allele, rt_allele diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 61071fc42..0a52012af 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -9,6 +9,7 @@ from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -27,10 +28,10 @@ async def test_populate_vep_e2e( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Enqueue the VEP job, run the worker, and verify consequence and annotation are populated.""" - _, mapped_variant = setup_sample_variants_for_vep + """Enqueue the VEP job, run the worker, and verify the allele's consequence and annotation.""" + variant, allele = setup_sample_alleles_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -42,13 +43,18 @@ async def test_populate_vep_e2e( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None annotation = session.scalars( select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, VariantAnnotationStatus.current.is_(True), ) @@ -62,7 +68,7 @@ async def test_populate_vep_e2e_with_recoder_path( arq_worker, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_protein_variant_for_vep, + setup_sample_protein_allele_for_vep, ): """VEP job uses Variant Recoder for a protein HGVS (NP_ accession) that VEP cannot resolve directly. @@ -70,7 +76,7 @@ async def test_populate_vep_e2e_with_recoder_path( does not return a consequence for. The job must fall back to Variant Recoder, recode it to a genomic HGVS, and then re-query VEP with the recoded string. """ - _, mapped_variant = setup_sample_protein_variant_for_vep + variant, allele = setup_sample_protein_allele_for_vep await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() @@ -82,13 +88,18 @@ async def test_populate_vep_e2e_with_recoder_path( session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence is not None - assert mapped_variant.vep_access_date is not None + live = session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele.id, + VepAlleleConsequence.current, + ) + ).one() + assert live.functional_consequence is not None + assert live.access_date is not None annotation = session.scalars( select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, + VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, VariantAnnotationStatus.current.is_(True), ) diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 611c33894..7e33f24fa 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -4,29 +4,48 @@ pytest.importorskip("arq") +from datetime import date, timedelta from unittest.mock import patch from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, JobStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") +_RESOLVE = "mavedb.worker.jobs.external_services.vep._resolve_consequences" +_RELEASE = "mavedb.worker.jobs.external_services.vep.get_ensembl_release" +_ENSEMBL_RELEASE = "116" + + +@pytest.fixture(autouse=True) +def mock_ensembl_release(): + """Stamp every job run with a fixed Ensembl release so tests version-key deterministically without + hitting /info/software. Tests exercising a release-fetch failure override this with an inner patch.""" + with patch(_RELEASE, return_value=_ENSEMBL_RELEASE): + yield + + +def _live_consequences_for(session, allele_id): + return session.scalars( + select(VepAlleleConsequence).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.current, + ) + ).all() + @pytest.mark.asyncio @pytest.mark.unit class TestPopulateVepForScoreSetUnit: - """Unit tests for populate_vep_for_score_set.""" + """Unit tests for the populate_vep_for_score_set job.""" - async def test_no_mapped_variants( + async def test_no_alleles_with_hgvs( self, session, with_populated_domain_data, @@ -34,7 +53,7 @@ async def test_no_mapped_variants( mock_worker_ctx, sample_populate_vep_run, ): - """Job succeeds with zero counts when no mapped variants exist.""" + """No alleles with HGVS -> the job succeeds with nothing to do.""" result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -43,405 +62,411 @@ async def test_no_mapped_variants( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 - assert result.data["variants_with_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 0 + assert result.data["created_allele_count"] == 0 + assert result.data["preexisting_allele_count"] == 0 + assert result.data["skipped_allele_count"] == 0 - async def test_variant_without_hgvs_assay_level_skipped( + async def test_calls_resolver_when_alleles_present( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A mapped variant with no hgvs_assay_level gets a SKIPPED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - mapped_variant.hgvs_assay_level = None - session.commit() - - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" + with patch(_RESOLVE, return_value={}) as mock_resolve: + result = await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 0 + mock_resolve.assert_called_once() - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SKIPPED - assert annotation.failure_category == AnnotationFailureCategory.MISSING_IDENTIFIER - - async def test_vep_api_success_sets_consequence_and_annotation( + async def test_propagates_exceptions( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """VEP returns a consequence: mapped variant and SUCCESS annotation are updated.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, + """Exceptions raised while resolving consequences are propagated.""" + with patch(_RESOLVE, side_effect=Exception("Test exception")): + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "Test exception" + + async def test_aborts_when_release_unavailable( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """If the Ensembl release cannot be fetched the job aborts rather than mis-versioning its writes + — the version is load-bearing for the skip, so a failure must propagate, not be swallowed.""" + with ( + patch(_RELEASE, side_effect=Exception("info/software unavailable")), + patch(_RESOLVE) as mock_resolve, ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with pytest.raises(Exception) as exc_info: + await populate_vep_for_score_set( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + ) + + assert str(exc_info.value) == "info/software unavailable" + mock_resolve.assert_not_called() # never queried VEP without a version to stamp + +@pytest.mark.asyncio +@pytest.mark.integration +class TestPopulateVepForScoreSetIntegration: + """Integration tests for the populate_vep_for_score_set job.""" + + async def test_no_alleles_with_hgvs( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + ): + """End-to-end: no alleles -> no consequence rows, no annotations, job succeeds.""" + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_processed"] == 1 - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" - assert mapped_variant.vep_access_date is not None - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS - async def test_vep_missing_triggers_variant_recoder_fallback( + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + + async def test_no_consequence_resolved( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When VEP misses a variant, Variant Recoder is called and its result fed back to VEP.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """An allele with HGVS that VEP cannot classify gets no consequence row and a SKIPPED VAS.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # initial VEP pass returns nothing - {genomic_hgvs: "missense_variant"}, # second VEP pass on recoded HGVS - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "skipped" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - async def test_variant_recoder_failure_annotated_as_failed( + async def test_successful_linking_independent( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Variant Recoder returning no result for an HGVS produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep + """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" + _, allele = setup_sample_alleles_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={}, # recoder has no result - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 0 - assert result.data["variants_recoder_failed"] == 1 + assert result.data["created_allele_count"] == 1 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE + assert live[0].access_date == date.today() - async def test_vep_failure_after_recoder_annotated_as_failed( + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + assert annotation_statuses[0].annotation_metadata["action"] == "created" + + async def test_links_rt_derived_allele_but_annotates_only_authoritative( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_rt_derived_allele_for_vep, ): - """VEP returning no consequence even after Variant Recoder produces a FAILED annotation.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + """VEP linkage must cover the full allele set, not just authoritative links: the RT-derived + allele's genomic HGVS resolves and must get a consequence row. Per-variant VAS, however, is + written only for the authoritative link (the interim bandaid) — so exactly one annotation row + is produced, keyed to the variant, never a second row for the RT-derived allele.""" + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, # VEP returns nothing in both passes - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding + # HGVS yields nothing this run. + with patch(_RESOLVE, return_value={rt_allele.hgvs_g: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_without_consequences"] == 1 - assert result.data["variants_recoder_failed"] == 0 - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.FAILED - assert annotation.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND - - async def test_vep_none_consequence_routes_to_recoder( + # The RT-derived (non-authoritative) allele IS linked — the core fix. + rt_live = _live_consequences_for(session, rt_allele.id) + assert len(rt_live) == 1 + assert rt_live[0].functional_consequence == "missense_variant" + # The authoritative allele's HGVS had no consequence, so it gets no row. + assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 + + # Annotation status is written only for the authoritative link: exactly one VAS row, for the + # variant. (Its status is "skipped" because the variant's authoritative allele had no + # consequence — cross-level resolution onto the RT-derived allele is deferred to AnnotationEvent.) + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].variant_id == variant.id + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + + async def test_skips_allele_already_at_current_release( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """A None consequence from VEP Phase 1 is treated as a miss and routed through Recoder. + """An allele with a live consequence already at the current Ensembl release is skipped: no VEP + query, the status reports SUCCESS/preexisting, and the existing row is not churned.""" + _, allele = setup_sample_alleles_for_vep + + # Simulate a prior run at the current release. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today(), + ) + ) + session.commit() - VEP can return an entry with most_severe_consequence=None when it recognises a variant - but cannot classify it. This should not silently fall into the UNKNOWN outcome branch — - instead it should be treated identically to an absent entry and sent to Recoder. - """ - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_hgvs = "NC_000017.11:g.43094692C>T" + with patch(_RESOLVE) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {hgvs: None}, # VEP knows the variant but returns no consequence - {genomic_hgvs: "missense_variant"}, # Phase 3 on recoded genomic string - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_hgvs]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + mock_resolve.assert_not_called() # version-keyed skip avoided the external query entirely + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - assert result.status == JobStatus.SUCCEEDED - assert result.data["variants_with_consequences"] == 1 + # Row not churned: still exactly one, still live. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" - async def test_most_severe_consequence_selected_from_multiple_genomic_hgvs( + async def test_new_release_same_consequence_bumps_in_place( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """When Recoder returns multiple genomic strings with different consequences, the most severe wins.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - genomic_less_severe = "NC_000017.11:g.43094692C>T" # → missense_variant - genomic_more_severe = "NC_000017.11:g.43094692C>A" # → stop_gained - - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=[ - {}, # Phase 1: VEP misses - { - genomic_less_severe: "missense_variant", - genomic_more_severe: "stop_gained", - }, # Phase 3: two different consequences - ], - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - return_value={hgvs: [genomic_less_severe, genomic_more_severe]}, - ), - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; an unchanged consequence advances + source_version (and access_date) in place — no supersede, so a new release does not churn + history for a categorical value that did not change.""" + _, allele = setup_sample_alleles_for_vep + + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_with_consequences"] == 1 + mock_resolve.assert_called_once() # older release -> not skipped, re-queried + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "stop_gained" + # Unchanged value -> no supersede: one row, still live, version + date advanced in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].source_version == _ENSEMBL_RELEASE + assert rows[0].access_date == date.today() - async def test_multiple_variants_sharing_hgvs_all_get_consequence( + async def test_force_requeries_unchanged_without_churn( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """All mapped variants that share an HGVS string each receive the consequence.""" - _, mapped_variant_1 = setup_sample_variants_for_vep - hgvs = mapped_variant_1.hgvs_assay_level - - score_set = session.get(ScoreSet, sample_populate_vep_run.job_params["score_set_id"]) - variant_2 = Variant( - urn="urn:variant:test-variant-for-vep-2", - score_set_id=score_set.id, - hgvs_nt=hgvs, - data={"hgvs_c": hgvs}, + """force bypasses the current-release skip and re-queries, but the linker only supersedes on a + value change: a forced re-run that resolves the same consequence reports preexisting and does + not churn the row (version/access_date advanced in place).""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence already at the current release (would be skipped without force). + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="missense_variant", + source_version=_ENSEMBL_RELEASE, + access_date=date.today() - timedelta(days=1), + ) ) - session.add(variant_2) session.commit() - mapped_variant_2 = MappedVariant( - variant_id=variant_2.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - post_mapped={"type": "Allele", "expressions": [{"value": hgvs, "syntax": "hgvs.c"}]}, - hgvs_assay_level=hgvs, - ) - session.add(mapped_variant_2) + + sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): - result = await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) - assert result.data["variants_processed"] == 2 - assert result.data["variants_with_consequences"] == 2 + mock_resolve.assert_called_once() # force bypassed the current-release skip + assert result.data["preexisting_allele_count"] == 1 + assert result.data["created_allele_count"] == 0 - session.refresh(mapped_variant_1) - session.refresh(mapped_variant_2) - assert mapped_variant_1.vep_functional_consequence == "missense_variant" - assert mapped_variant_2.vep_functional_consequence == "missense_variant" + # Unchanged consequence -> no supersede: one row, still live, access_date touched in place. + rows = session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id)).all() + assert len(rows) == 1 + assert rows[0].valid_to is None + assert rows[0].access_date == date.today() - async def test_vep_batch_api_exception_raises( + async def test_new_release_changed_consequence_supersedes( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the VEP API propagates to the job management decorator.""" - with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - side_effect=RuntimeError("VEP API unreachable"), - ), - pytest.raises(RuntimeError, match="VEP API unreachable"), - ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), + """An allele live at an older release is re-queried; a changed result supersedes it — one live + row carrying the new consequence/release and one retired row preserving the old.""" + _, allele = setup_sample_alleles_for_vep + + # Prior live consequence at an older release -> not skipped, eligible for re-query. + session.add( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence="synonymous_variant", + source_version="115", + access_date=date.today() - timedelta(days=90), ) + ) + session.commit() + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.data["created_allele_count"] == 1 + + live = _live_consequences_for(session, allele.id) + assert len(live) == 1 + assert live[0].functional_consequence == "missense_variant" + assert live[0].source_version == _ENSEMBL_RELEASE - async def test_variant_recoder_api_exception_raises( + # Old consequence retired, not deleted. + all_rows = session.scalars( + select(VepAlleleConsequence).where(VepAlleleConsequence.allele_id == allele.id) + ).all() + assert len(all_rows) == 2 + assert len([r for r in all_rows if r.valid_to is not None]) == 1 + + async def test_successful_linking_pipeline( + self, + session, + with_populated_domain_data, + mock_worker_ctx, + sample_populate_vep_run_pipeline, + sample_populate_vep_pipeline, + setup_sample_alleles_for_vep, + ): + """End-to-end successful linking within a pipeline updates both job and pipeline status.""" + _, allele = setup_sample_alleles_for_vep + + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) + + assert result.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_exceptions_handled_by_decorators( self, session, with_populated_domain_data, with_populate_vep_job, mock_worker_ctx, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """An unexpected exception from the Variant Recoder API propagates to the job management decorator.""" + """Exceptions during resolution are handled by the job decorators (Slack alert, ERRORED).""" with ( - patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={}, - ), - patch( - "mavedb.worker.jobs.external_services.vep.run_variant_recoder", - side_effect=RuntimeError("Recoder API unreachable"), - ), - pytest.raises(RuntimeError, match="Recoder API unreachable"), + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, ): - await populate_vep_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_vep_run.id), - ) + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + mock_send_slack_job_error.assert_called_once() + assert isinstance(result, JobExecutionOutcome) + assert result.status == JobStatus.ERRORED + assert isinstance(result.exception, Exception) + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED @pytest.mark.asyncio @pytest.mark.integration -class TestPopulateVepForScoreSetIntegration: - """Integration tests for populate_vep_for_score_set run through an ARQ worker context.""" +class TestPopulateVepForScoreSetArqContext: + """Tests for the populate_vep_for_score_set job using the ARQ context fixture.""" - async def test_populate_vep_with_arq_context( + async def test_populate_vep_with_arq_context_independent( self, arq_redis, arq_worker, @@ -449,35 +474,27 @@ async def test_populate_vep_with_arq_context( with_populated_domain_data, with_populate_vep_job, sample_populate_vep_run, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes successfully within an ARQ worker context.""" - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level + """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "missense_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() - session.refresh(sample_populate_vep_run) - assert sample_populate_vep_run.status == JobStatus.SUCCEEDED + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - session.refresh(mapped_variant) - assert mapped_variant.vep_functional_consequence == "missense_variant" + annotation_statuses = session.query(VariantAnnotationStatus).all() + assert len(annotation_statuses) == 1 + assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - ).one() - assert annotation.status == AnnotationStatus.SUCCESS + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.SUCCEEDED - async def test_populate_vep_in_pipeline_context( + async def test_populate_vep_with_arq_context_pipeline( self, arq_redis, arq_worker, @@ -485,24 +502,73 @@ async def test_populate_vep_in_pipeline_context( with_populated_domain_data, sample_populate_vep_run_pipeline, sample_populate_vep_pipeline, - setup_sample_variants_for_vep, + setup_sample_alleles_for_vep, ): - """Job completes and advances the pipeline when run in a pipeline context.""" - from mavedb.models.enums.job_pipeline import PipelineStatus + """The VEP job completes and advances the pipeline through the ARQ worker.""" + _, allele = setup_sample_alleles_for_vep - _, mapped_variant = setup_sample_variants_for_vep - hgvs = mapped_variant.hgvs_assay_level - - with patch( - "mavedb.worker.jobs.external_services.vep.get_functional_consequence", - return_value={hgvs: "synonymous_variant"}, - ): + with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() + assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 + session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED - session.refresh(sample_populate_vep_pipeline) assert sample_populate_vep_pipeline.status == PipelineStatus.SUCCEEDED + + async def test_populate_vep_with_arq_context_exception_handling_independent( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + with_populate_vep_job, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job are handled with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run) + assert sample_populate_vep_run.status == JobStatus.ERRORED + + async def test_populate_vep_with_arq_context_exception_handling_pipeline( + self, + arq_redis, + arq_worker, + session, + with_populated_domain_data, + sample_populate_vep_pipeline, + sample_populate_vep_run_pipeline, + setup_sample_alleles_for_vep, + ): + """Exceptions in the VEP job fail the pipeline with the ARQ context fixture.""" + with ( + patch(_RESOLVE, side_effect=Exception("Test exception")), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, + ): + await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) + await arq_worker.async_run() + await arq_worker.run_check() + + mock_send_slack_job_error.assert_called_once() + assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 + assert len(session.query(VariantAnnotationStatus).all()) == 0 + + session.refresh(sample_populate_vep_run_pipeline) + assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED + session.refresh(sample_populate_vep_pipeline) + assert sample_populate_vep_pipeline.status == PipelineStatus.FAILED From 62476807d001c7dffee473331cedc5d68c9bb362 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 16:52:47 -0700 Subject: [PATCH 27/93] feat(clinvar): migrate ClinVar annotation onto the allele model Step 3 of the #742 external-annotation migration. ClinVar linkage moves off MappedVariant onto the deduplicated allele model. - Rename clinical_controls -> clinvar_controls (ORM model + table + unique constraint). Internal only: the ClinicalControl* view models, the /clinical-controls serving endpoints, and the frozen mapped_variants_clinical_controls association are unchanged, since the view-model name is both the OpenAPI schema name and the record_type discriminator consumed by the UI. - New clinvar_allele_links ValidTime table + ClinvarAlleleLink model. Multi-live: partial unique index (allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live link per release. - Refactor refresh_clinvar_controls onto get_alleles_for_score_set + group_alleles_for_annotation (payload = CAID, full allele scope). Links are get-or-create; a same-version re-resolution to a different control supersedes newest-wins (gap-free retire+insert) rather than leaving two live links. VAS writes funnel through the _annotate_clinvar choke point, fanned only to authoritative_variant_ids and version-scoped. - Additively capture ClinVar's VariationID: nullable clinvar_variation_id column populated forward from the variant_summary TSV (parse degrades to None on archival schemas lacking the column). Unserved; the dedicated clinvar_variants remodel is deferred to the read-cutover. Tests rewritten for the allele model, covering the multi-live link writes, the version-scoped supersede guard, and the authoritative-only VAS fan-out. --- ...e_clinical_controls_to_clinvar_controls.py | 39 + .../c3d4e5f6a7b8_add_clinvar_allele_links.py | 73 + .../d4e5f6a7b8c9_add_clinvar_variation_id.py | 29 + src/mavedb/lib/clinvar/constants.py | 6 +- src/mavedb/lib/clinvar/utils.py | 5 +- src/mavedb/lib/score_sets.py | 24 +- src/mavedb/models/__init__.py | 1 + src/mavedb/models/clinical_control.py | 20 +- .../models/clinical_control_mapped_variant.py | 2 +- src/mavedb/models/clinvar_allele_link.py | 65 + src/mavedb/models/mapped_variant.py | 6 +- src/mavedb/routers/score_sets.py | 20 +- .../worker/jobs/external_services/clinvar.py | 346 ++-- tests/helpers/util/score_set.py | 2 +- tests/routers/conftest.py | 6 +- .../external_services/network/test_clinvar.py | 13 +- .../jobs/external_services/test_clinvar.py | 1413 +++-------------- 17 files changed, 734 insertions(+), 1336 deletions(-) create mode 100644 alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py create mode 100644 alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py create mode 100644 alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py create mode 100644 src/mavedb/models/clinvar_allele_link.py diff --git a/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py new file mode 100644 index 000000000..19cca0789 --- /dev/null +++ b/alembic/versions/b2c3d4e5f6a7_rename_clinical_controls_to_clinvar_controls.py @@ -0,0 +1,39 @@ +"""rename clinical_controls to clinvar_controls + +Revision ID: b2c3d4e5f6a7 +Revises: a7c4e9d2f1b8 +Create Date: 2026-06-22 + +Renames the clinical_controls entity table to clinvar_controls, and renames the unique +constraint to match. The frozen association table (mapped_variants_clinical_controls) +and its FK to the renamed table are left structurally intact — PostgreSQL updates the FK +target automatically on table rename. The Python model is renamed ClinicalControl → +ClinvarControl in the same changeset (no data migration). +""" + +from alembic import op + +revision = "b2c3d4e5f6a7" +down_revision = "a7c4e9d2f1b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.rename_table("clinical_controls", "clinvar_controls") + # PostgreSQL does not auto-rename constraints on table rename; rename explicitly so the + # on_conflict_do_update(constraint=...) in the job references the correct name. + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinical_controls_db_name_identifier_version " + "TO uq_clinvar_controls_db_name_identifier_version" + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE clinvar_controls RENAME CONSTRAINT " + "uq_clinvar_controls_db_name_identifier_version " + "TO uq_clinical_controls_db_name_identifier_version" + ) + op.rename_table("clinvar_controls", "clinical_controls") diff --git a/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py new file mode 100644 index 000000000..4a20f008f --- /dev/null +++ b/alembic/versions/c3d4e5f6a7b8_add_clinvar_allele_links.py @@ -0,0 +1,73 @@ +"""add clinvar_allele_links table + +Revision ID: c3d4e5f6a7b8 +Revises: b2c3d4e5f6a7 +Create Date: 2026-06-22 + +New valid-time link table connecting deduplicated alleles to ClinvarControl rows, replacing +the frozen mapped_variants_clinical_controls association for new-model writes. + +ClinVar's link shape is deliberately multi-live: the partial unique index is +(allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live +link per ClinVar release rather than superseding as in gnomAD/VEP. Each ClinVar release is a +distinct ClinvarControl row, so different releases stack as independent live links. A link is +retired (valid_to closed) only if ClinVar later removes the variant from a release, which +would surface as a re-run finding no data for that release and retiring the corresponding +link — archival data never changes, so this path is theoretical. + +The existing mapped_variants_clinical_controls association table is left untouched (frozen +for serving existing data). +""" + +import sqlalchemy as sa + +from alembic import op + +revision = "c3d4e5f6a7b8" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "clinvar_allele_links", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("allele_id", sa.Integer(), nullable=False), + sa.Column("clinvar_control_id", sa.Integer(), nullable=False), + sa.Column("valid_from", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("valid_to", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["allele_id"], + ["alleles.id"], + name="fk_clinvar_allele_links_allele_id", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["clinvar_control_id"], + ["clinvar_controls.id"], + name="fk_clinvar_allele_links_clinvar_control_id", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_clinvar_allele_links_allele_id", "clinvar_allele_links", ["allele_id"]) + op.create_index("ix_clinvar_allele_links_clinvar_control_id", "clinvar_allele_links", ["clinvar_control_id"]) + # Multi-live: one live link per (allele, release). An allele accumulates one live link per + # ClinVar release rather than superseding — unlike gnomAD/VEP which enforce one live link + # per allele across all versions. Superseded rows (valid_to IS NOT NULL) are preserved for + # point-in-time queries. + op.create_index( + "uq_clinvar_allele_links_live", + "clinvar_allele_links", + ["allele_id", "clinvar_control_id"], + unique=True, + postgresql_where=sa.text("valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_clinvar_allele_links_live", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_clinvar_control_id", table_name="clinvar_allele_links") + op.drop_index("ix_clinvar_allele_links_allele_id", table_name="clinvar_allele_links") + op.drop_table("clinvar_allele_links") diff --git a/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py new file mode 100644 index 000000000..de5758b4c --- /dev/null +++ b/alembic/versions/d4e5f6a7b8c9_add_clinvar_variation_id.py @@ -0,0 +1,29 @@ +"""add clinvar_variation_id to clinvar_controls + +Revision ID: d4e5f6a7b8c9 +Revises: c3d4e5f6a7b8 +Create Date: 2026-06-22 + +Additive, non-breaking column for ClinVar's canonical public identifier (VariationID), captured forward +from the variant_summary TSV. db_identifier continues to hold the AlleleID (the allele-level handle used +for gnomAD cross-references); this carries the VariationID beside it for eventual external ClinVar links +(clinvar/variation/{id}). Nullable and not yet served — the dedicated clinvar_variants remodel (explicit +fields replacing the generic db_* shape, the serving/UI cutover, and backfill of existing rows) is +deferred. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c3d4e5f6a7b8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("clinvar_controls", sa.Column("clinvar_variation_id", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("clinvar_controls", "clinvar_variation_id") diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..fd80212be 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -26,6 +26,10 @@ backoff for throttling is unnecessary — a modest retry with short backoff suffices. """ -CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus") +CLINVAR_FIELDS_TO_KEEP = ("GeneSymbol", "ClinicalSignificance", "ReviewStatus", "VariationID") """Only these fields are extracted from each ClinVar TSV row and cached. The full TSV has ~30 columns; trimming to only what we need shrinks the cached pickle from hundreds of MB to tens of MB and speeds up load times. + +VariationID is ClinVar's canonical public identifier (anchors the web UI / variation links); we keep it +alongside the AlleleID (the row key) so the link record can carry both. A row missing the column on an +older archival TSV degrades to None rather than failing the whole version's parse. """ diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..d72968da8 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -175,8 +175,11 @@ def _fetch_parse_and_cache( # as a list (which would be 1.5–2 GB for a modern TSV). with gzip.open(filename=buf, mode="rt") as f: reader = csv.DictReader(f, delimiter="\t") # type: ignore + # row.get (not row[field]) so a field absent from an older archival TSV schema yields + # None for that row rather than raising and discarding the whole version's parse. data: Dict[str, Dict[str, str]] = { - str(row["#AlleleID"]): {field: row[field] for field in CLINVAR_FIELDS_TO_KEEP} for row in reader + str(row["#AlleleID"]): {field: row.get(field) for field in CLINVAR_FIELDS_TO_KEEP} # type: ignore[misc] + for row in reader } finally: csv.field_size_limit(default_csv_field_size_limit) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 2acfa26f7..76cbab9e6 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -38,7 +38,7 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant @@ -734,14 +734,14 @@ def get_score_set_variants_as_csv( idx = 2 if need_mappings else 1 gnomad_data.append(row[idx]) - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinvarControl. + clinvar_data_map: dict[str, dict[int, Optional[ClinvarControl]]] = {} if clinvar_namespaces and mappings is not None: mv_ids = [m.id for m in mappings if m is not None] for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + mv_to_cc: dict[int, Optional[ClinvarControl]] = {} if mv_ids: - aliased_cc = aliased(ClinicalControl) + aliased_cc = aliased(ClinvarControl) cc_query = ( select( mapped_variants_clinical_controls_association_table.c.mapped_variant_id, @@ -764,11 +764,11 @@ def get_score_set_variants_as_csv( clinvar_data_map[ns] = mv_to_cc # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = None if clinvar_namespaces and mappings is not None: clinvar_per_variant = [] for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} + row_clinvar: dict[str, Optional[ClinvarControl]] = {} for ns, mv_to_cc in clinvar_data_map.items(): if mapping is not None and mapping.id is not None: row_clinvar[ns] = mv_to_cc.get(mapping.id) @@ -857,7 +857,7 @@ def variant_to_csv_row( columns: dict[str, list[str]], mapping: Optional[MappedVariant] = None, gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, namespaced: Optional[bool] = None, na_rep="NA", ) -> dict[str, Any]: @@ -876,7 +876,7 @@ def variant_to_csv_row( Mapped variant corresponding to the variant. gnomad_data : variant.models.GnomADVariant, optional gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional + clinvar_data_by_ns : dict[str, Optional[ClinvarControl]], optional Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). na_rep : str String to represent null values. @@ -1005,7 +1005,7 @@ def variants_to_csv_rows( columns: dict[str, list[str]], mappings: Optional[Sequence[Optional[MappedVariant]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, namespaced: Optional[bool] = None, na_rep="NA", ) -> Iterable[dict[str, Any]]: @@ -1024,7 +1024,7 @@ def variants_to_csv_rows( List of mapped variants corresponding to the variants. gnomad_data : list[Optional[variant.models.GnomADVariant]], optional List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional + clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinvarControl]]]], optional Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). na_rep : str String to represent null values. @@ -1036,7 +1036,7 @@ def variants_to_csv_rows( n = len(variants) _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n ) return map( diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index b18e20ea7..c181e4099 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -4,6 +4,7 @@ "allele", "collection", "clinical_control", + "clinvar_allele_link", "controlled_keyword", "doi_identifier", "ensembl_identifier", diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 0b989cb22..cf28b5b8b 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -8,18 +8,19 @@ from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table if TYPE_CHECKING: + from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.mapped_variant import MappedVariant -class ClinicalControl(Base): - __tablename__ = "clinical_controls" +class ClinvarControl(Base): + __tablename__ = "clinvar_controls" __table_args__ = ( UniqueConstraint( - "db_name", "db_identifier", "db_version", name="uq_clinical_controls_db_name_identifier_version" + "db_name", "db_identifier", "db_version", name="uq_clinvar_controls_db_name_identifier_version" ), ) - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) gene_symbol = Column(String, nullable=False, index=True) @@ -27,14 +28,25 @@ class ClinicalControl(Base): clinical_review_status = Column(String, nullable=False) db_name = Column(String, nullable=False, index=True) + # Currently the ClinVar Allele ID. db_identifier = Column(String, nullable=False, index=True) db_version = Column(String, nullable=False, index=True) + # ClinVar's canonical public identifier (ClinVar Variation ID). + clinvar_variation_id = Column(String, nullable=True) + creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) + # Frozen serving path: links to MappedVariant via the old association table (never written for new data). mapped_variants: Mapped[list["MappedVariant"]] = relationship( "MappedVariant", secondary=mapped_variants_clinical_controls_association_table, back_populates="clinical_controls", ) + + # New-model annotation links (one live link per allele per ClinVar release). + allele_links: Mapped[list["ClinvarAlleleLink"]] = relationship( + "ClinvarAlleleLink", + back_populates="clinvar_control", + ) diff --git a/src/mavedb/models/clinical_control_mapped_variant.py b/src/mavedb/models/clinical_control_mapped_variant.py index eabb7689a..b7fc5d84c 100644 --- a/src/mavedb/models/clinical_control_mapped_variant.py +++ b/src/mavedb/models/clinical_control_mapped_variant.py @@ -7,5 +7,5 @@ "mapped_variants_clinical_controls", Base.metadata, Column("mapped_variant_id", ForeignKey("mapped_variants.id"), primary_key=True), - Column("clinical_control_id", ForeignKey("clinical_controls.id"), primary_key=True), + Column("clinical_control_id", ForeignKey("clinvar_controls.id"), primary_key=True), ) diff --git a/src/mavedb/models/clinvar_allele_link.py b/src/mavedb/models/clinvar_allele_link.py new file mode 100644 index 000000000..941ee2f22 --- /dev/null +++ b/src/mavedb/models/clinvar_allele_link.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING + +from sqlalchemy import Column, ForeignKey, Index, Integer, text +from sqlalchemy.orm import Mapped, relationship + +from mavedb.db.base import Base +from mavedb.db.mixins import ValidTime + +if TYPE_CHECKING: + from .allele import Allele + from .clinical_control import ClinvarControl + + +class ClinvarAlleleLink(ValidTime, Base): + """Valid-time link between an :class:`Allele` and a :class:`ClinvarControl` release. + + Replaces the frozen ``mapped_variants_clinical_controls`` association table for new-model writes. + A link is live while ``valid_to`` is NULL. Unlike gnomAD/VEP (one live result per allele), the partial + unique index is ``(allele_id, clinvar_control_id) WHERE valid_to IS NULL`` — **multi-live**: an allele + accumulates one live link per ClinVar release, because each release is a distinct, versioned + ``ClinvarControl`` assertion that stacks rather than supersedes. A link retires only on two theoretical + paths (archival data does not change): ClinVar drops the variant from a release (a re-run finds no data + for it), or the allele re-resolves to a *different* control within the same release — the job supersedes + that newest-wins to preserve one live link per (allele, release), since this index only enforces one live + link per (allele, control). + """ + + __tablename__ = "clinvar_allele_links" + + id: Mapped[int] = Column(Integer, primary_key=True) + allele_id: Mapped[int] = Column( + Integer, + ForeignKey("alleles.id", ondelete="RESTRICT"), + nullable=False, + ) + clinvar_control_id: Mapped[int] = Column( + Integer, + ForeignKey("clinvar_controls.id", ondelete="RESTRICT"), + nullable=False, + ) + + # One-directional to Allele (no reverse collection there); back-ref on the ClinvarControl entity only. + allele: Mapped["Allele"] = relationship("Allele") + clinvar_control: Mapped["ClinvarControl"] = relationship("ClinvarControl", back_populates="allele_links") + + __table_args__ = ( + Index( + "ix_clinvar_allele_links_allele_id", + "allele_id", + ), + Index( + "ix_clinvar_allele_links_clinvar_control_id", + "clinvar_control_id", + ), + # Multi-live: one live link per (allele, release). Each ClinVar release is a distinct + # ClinvarControl row, so different releases stack as independent live links rather than + # superseding. Only live rows participate in this constraint. + Index( + "uq_clinvar_allele_links_live", + "allele_id", + "clinvar_control_id", + unique=True, + postgresql_where=text("valid_to IS NULL"), + ), + ) diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index b35f1bbb4..40325cf17 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -11,7 +11,7 @@ from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: - from .clinical_control import ClinicalControl + from .clinical_control import ClinvarControl from .gnomad_variant import GnomADVariant from .target_gene_mapping import TargetGeneMapping from .variant import Variant @@ -61,8 +61,8 @@ class MappedVariant(Base): hgvs_c = Column(String, nullable=True) hgvs_p = Column(String, nullable=True) - clinical_controls: Mapped[list["ClinicalControl"]] = relationship( - "ClinicalControl", + clinical_controls: Mapped[list["ClinvarControl"]] = relationship( + "ClinvarControl", secondary=mapped_variants_clinical_controls_association_table, back_populates="mapped_variants", ) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 371862d15..04f798a70 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -73,7 +73,7 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment @@ -2411,7 +2411,7 @@ async def get_clinical_controls_for_score_set( user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinicalControl]: +) -> Sequence[ClinvarControl]: """ Fetch relevant clinical controls for a given score set. """ @@ -2432,23 +2432,23 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinicalControl) - .join(ClinicalControl.mapped_variants) + select(ClinvarControl) + .join(ClinvarControl.mapped_variants) .join(MappedVariant.variant) - .options(contains_eager(ClinicalControl.mapped_variants).contains_eager(MappedVariant.variant)) + .options(contains_eager(ClinvarControl.mapped_variants).contains_eager(MappedVariant.variant)) .filter(MappedVariant.current.is_(True)) .filter(Variant.score_set_id == item.id) ) if db_name is not None: save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_name == db_name) + clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_name == db_name) if db_version is not None: save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinicalControl.db_version == db_version) + clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_version == db_version) - clinical_controls: Sequence[ClinicalControl] = _db.scalars(clinical_controls_query).unique().all() + clinical_controls: Sequence[ClinvarControl] = _db.scalars(clinical_controls_query).unique().all() if not clinical_controls: logger.info( @@ -2496,8 +2496,8 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinicalControl.db_name, ClinicalControl.db_version) - .join(MappedVariant, ClinicalControl.mapped_variants) + select(ClinvarControl.db_name, ClinvarControl.db_version) + .join(MappedVariant, ClinvarControl.mapped_variants) .join(Variant) .where(MappedVariant.current.is_(True)) .where(Variant.score_set_id == item.id) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 68a34a04b..8041d14f7 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -1,35 +1,33 @@ -"""ClinVar integration jobs for variant annotation +"""ClinVar integration jobs for variant annotation. -This module contains job definitions and utility functions for integrating ClinVar -variant data into MaveDB. It includes functions to fetch and parse ClinVar variant -summary data, and update MaveDB records with the latest ClinVar annotations. +Links deduplicated alleles to ClinVar clinical-control data across every archival ClinVar +release. Each release is a distinct, versioned ``ClinvarControl`` entity, and an allele +accumulates one live ``ClinvarAlleleLink`` per release it appears in (multi-live, unlike +gnomAD/VEP which hold one live result per allele). Both ClinGen API calls and ClinVar TSV data fetches are automatically cached using aiocache with Redis backend: - ClinGen API calls: 24-hour TTL - ClinVar TSV files: 90-day TTL (archival data doesn't change) - -This significantly reduces redundant network requests when refreshing ClinVar -controls across multiple months/years. """ import logging from datetime import datetime import requests -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.dialects.postgresql import insert as pg_insert from mavedb.lib.annotation_status_manager import AnnotationStatusManager from mavedb.lib.clingen.allele_registry import get_associated_clinvar_allele_id +from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management from mavedb.worker.lib.managers.job_manager import JobManager @@ -42,7 +40,7 @@ CLINVAR_START_MONTH = 2 -def generate_clinvar_versions() -> list[tuple[int, int]]: +def _generate_clinvar_versions() -> list[tuple[int, int]]: """Generate all ClinVar version (year, month) pairs from Feb 2015 to current Jan. Returns a list of (year, month) tuples representing each ClinVar archival @@ -55,15 +53,64 @@ def generate_clinvar_versions() -> list[tuple[int, int]]: return versions +def _annotate_clinvar( + annotation_manager: AnnotationStatusManager, + variant_ids: list[int], + version: str, + status: AnnotationStatus, + *, + failure_category: AnnotationFailureCategory | None = None, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Fan a CLINVAR_CONTROL annotation out to every variant served by an allele. + + AAS migration seam: the single choke point for ClinVar's per-variant VAS writes. At migration it + becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant + association narrows to provenance. See docs/design/allele-annotation-status.md. + + Unlike gnomAD/VEP, ClinVar writes a status row per ClinVar release, so retirement is + version-scoped (``replace_all_versions=False``) — each release keeps its own current row. + """ + annotation_data: dict = {"annotation_metadata": metadata or {}} + if error_message is not None: + annotation_data["error_message"] = error_message + + for variant_id in variant_ids: + annotation_manager.add_annotation( + variant_id=variant_id, + annotation_type=AnnotationType.CLINVAR_CONTROL, + version=version, + status=status, + failure_category=failure_category, + annotation_data=annotation_data, + current=True, + replace_all_versions=False, + ) + + @with_pipeline_management async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Refresh ClinVar clinical control data across all archival versions. - - Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each - subsequent year through the current year), fetching TSV data and updating - clinical control records for all mapped variants in the score set. Individual - version failures are logged and skipped — the job continues processing - remaining versions. + """Link deduplicated alleles to ClinVar clinical-control data across all archival versions. + + Iterates over every ClinVar archival snapshot (Feb 2015, then Jan of each subsequent year through + the current year). For each version it resolves each allele's ClinGen Allele ID (CAID) to a ClinVar + allele id, upserts the versioned :class:`ClinvarControl`, and establishes a live + :class:`ClinvarAlleleLink`. An allele accumulates one live link per release (multi-live). Individual + version fetch failures are logged and skipped — the job continues with the remaining versions. + + Job Parameters: + - score_set_id (int): The ID of the ScoreSet whose alleles to process. + - correlation_id (str): Correlation ID for tracing requests across services. + - force (bool, optional): Bypass the per-version skip and re-resolve every allele. The link + write still get-or-creates (no duplicate live link), so a forced re-run of unchanged data + writes no new links. + + Side Effects: + - Creates ClinvarControl and ClinvarAlleleLink rows. + + Returns: + JobExecutionOutcome: outcome with version and per-link counts. """ job = job_manager.get_job() @@ -72,8 +119,9 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore correlation_id = job.job_params["correlation_id"] # type: ignore + force = bool(job.job_params.get("force", False)) # type: ignore[union-attr] - versions = generate_clinvar_versions() + versions = _generate_clinvar_versions() job_manager.save_to_context( { @@ -83,24 +131,53 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag "correlation_id": correlation_id, "versions": versions, "total_versions": len(versions), + "force": force, } ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - variants_to_refresh = job_manager.db.scalars( - select(MappedVariant) - .join(Variant) - .where( - Variant.score_set_id == score_set.id, - MappedVariant.current.is_(True), + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often + # the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + allele_data = group_alleles_for_annotation( + get_alleles_for_score_set(job_manager.db, score_set.id), + payload=lambda row: row.clingen_allele_id, + ) + job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + + if not allele_data: + logger.warning( + msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", + extra=job_manager.logging_context(), + ) + job_manager.db.flush() + return JobExecutionOutcome.succeeded( + data={ + "versions_completed": 0, + "versions_total": len(versions), + "created_link_count": 0, + "preexisting_link_count": 0, + } + ) + + all_allele_ids = set(allele_data.keys()) + + def alleles_linked_at_version(clinvar_version: str) -> set[int]: + """Allele ids (within the work set) holding a live ClinvarAlleleLink to a control of this version.""" + return set( + job_manager.db.scalars( + select(ClinvarAlleleLink.allele_id) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(all_allele_ids)) + .where(ClinvarAlleleLink.current) + .where(ClinvarControl.db_version == clinvar_version) + ).all() ) - ).all() - total_variants_to_refresh = len(variants_to_refresh) - job_manager.save_to_context({"total_variants_to_refresh": total_variants_to_refresh}) - total_refreshed = 0 total_failed = 0 + created_link_count = 0 + preexisting_link_count = 0 versions_completed = 0 for version_index, (year, month) in enumerate(versions): @@ -125,160 +202,175 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) continue + # Cost: skip alleles already linked at this version (an archival release cannot change). force + # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. + already_linked = set() if force else alleles_linked_at_version(clinvar_version) + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for mapped_variant in variants_to_refresh: - clingen_id = mapped_variant.clingen_allele_id - - if clingen_id is None: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "Mapped variant does not have an associated ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + for allele_id, entry in allele_data.items(): + caid = entry.payload + + if allele_id in already_linked: + preexisting_link_count += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "action": "preexisting"}, ) continue - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + if "," in caid: + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", - }, - current=True, - replace_all_versions=False, + error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", + metadata={"clingen_allele_id": caid}, ) continue try: - clinvar_allele_id = await get_associated_clinvar_allele_id(clingen_id) # type: ignore + clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.FAILED, + total_failed += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.FAILED, failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", - }, - current=True, - replace_all_versions=False, + error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", + metadata={"clingen_allele_id": caid}, ) logger.error( - f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {clingen_id}.", + f"Failed to retrieve ClinVar allele ID from ClinGen API for ClinGen allele ID {caid}.", extra=job_manager.logging_context(), exc_info=exc, ) - total_failed += 1 continue if not clinvar_allele_id: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": "No ClinVar allele ID found for ClinGen allele ID.", - }, - current=True, - replace_all_versions=False, + error_message="No ClinVar allele ID found for ClinGen allele ID.", + metadata={"clingen_allele_id": caid}, ) continue if clinvar_allele_id not in tsv_data: - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SKIPPED, + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SKIPPED, failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": "No ClinVar data found for ClinVar allele ID.", - }, - current=True, - replace_all_versions=False, + error_message="No ClinVar data found for ClinVar allele ID.", + metadata={"clingen_allele_id": caid}, ) continue variant_data = tsv_data[clinvar_allele_id] - identifier = str(clinvar_allele_id) - # Atomic upsert — avoids a check-then-act race when two - # refresh_clinvar_controls jobs run concurrently for different - # score sets and encounter the same (db_name, db_identifier, - # db_version) tuple. ON CONFLICT DO UPDATE is guaranteed to - # return exactly one row regardless of concurrent inserts. + # Atomic upsert — avoids a check-then-act race when two refresh_clinvar_controls jobs run + # concurrently for different score sets and encounter the same + # (db_name, db_identifier, db_version) tuple. ON CONFLICT DO UPDATE returns exactly one row. upsert_stmt = ( - pg_insert(ClinicalControl) + pg_insert(ClinvarControl) .values( - db_identifier=identifier, + db_identifier=str(clinvar_allele_id), db_version=clinvar_version, db_name="ClinVar", gene_symbol=variant_data.get("GeneSymbol"), clinical_significance=variant_data.get("ClinicalSignificance"), clinical_review_status=variant_data.get("ReviewStatus"), + clinvar_variation_id=variant_data.get("VariationID"), ) .on_conflict_do_update( - constraint="uq_clinical_controls_db_name_identifier_version", + constraint="uq_clinvar_controls_db_name_identifier_version", set_={ "gene_symbol": variant_data.get("GeneSymbol"), "clinical_significance": variant_data.get("ClinicalSignificance"), "clinical_review_status": variant_data.get("ReviewStatus"), + "clinvar_variation_id": variant_data.get("VariationID"), }, ) - .returning(ClinicalControl) + .returning(ClinvarControl) ) - clinvar_variant = job_manager.db.scalars(upsert_stmt).one() - - job_manager.db.add(clinvar_variant) - job_manager.db.flush() - - if clinvar_variant not in mapped_variant.clinical_controls: - mapped_variant.clinical_controls.append(clinvar_variant) - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=mapped_variant.variant_id, # type: ignore - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=clinvar_version, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "clinvar_allele_id": clinvar_allele_id, - }, - }, - current=True, - replace_all_versions=False, + clinvar_control = job_manager.db.scalars(upsert_stmt).one() + + # At most one live link per (allele, release). Normally a release is immutable, so the + # allele's live link for this version is either a reconfirm of this same control (no-op) or + # absent (insert) — multi-live accumulates only across *different* releases, never supersedes. + # Defensive guard: if the allele already holds a live link to a *different* control of this + # same version, the release re-resolved under us (re-ingestion / upstream correction — should + # never happen for archival data). Supersede newest-wins with a shared timestamp and log. + live_link = job_manager.db.scalar( + select(ClinvarAlleleLink) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.current, + ClinvarControl.db_version == clinvar_version, + ) ) + if live_link is None: + job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) + created_link_count += 1 + action = "created" + elif live_link.clinvar_control_id == clinvar_control.id: + preexisting_link_count += 1 + action = "preexisting" + else: + at = job_manager.db.scalar(select(func.now())) + assert at is not None # SELECT now() always returns a timestamp + live_link.retire(at=at) + job_manager.db.add( + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=at) + ) + created_link_count += 1 + action = "superseded" + logger.warning( + msg=( + f"Allele {allele_id} held a live ClinVar link to control " + f"{live_link.clinvar_control_id} for version {clinvar_version}, but re-resolved to " + f"control {clinvar_control.id} (ClinVar allele {clinvar_allele_id}). Superseding " + "newest-wins; archival ClinVar data should be immutable — investigate the upstream " + "re-resolution." + ), + extra=job_manager.logging_context(), + ) - total_refreshed += 1 + _annotate_clinvar( + annotation_manager, + entry.authoritative_variant_ids, + clinvar_version, + AnnotationStatus.SUCCESS, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id, "action": action}, + ) annotation_manager.flush() + job_manager.db.flush() versions_completed += 1 logger.info( - f"Completed ClinVar version {clinvar_version} for {total_variants_to_refresh} variants.", + f"Completed ClinVar version {clinvar_version} for {len(allele_data)} alleles.", extra=job_manager.logging_context(), ) logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{total_refreshed} variant-version annotations.", + f"{created_link_count} new links, {preexisting_link_count} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and total_refreshed == 0: + if total_failed > 0 and created_link_count == 0 and preexisting_link_count == 0: error_message = ( f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." ) @@ -289,7 +381,8 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": 0, + "created_link_count": 0, + "preexisting_link_count": 0, }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) @@ -299,6 +392,7 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag data={ "versions_completed": versions_completed, "versions_total": len(versions), - "variant_annotations": total_refreshed, + "created_link_count": created_link_count, + "preexisting_link_count": preexisting_link_count, } ) diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index b6d7801ae..83ebcbca8 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinicalControl as ClinicalControlDbModel +from mavedb.models.clinical_control import ClinvarControl as ClinicalControlDbModel from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel diff --git a/tests/routers/conftest.py b/tests/routers/conftest.py index ba34c5489..0971bde47 100644 --- a/tests/routers/conftest.py +++ b/tests/routers/conftest.py @@ -3,7 +3,7 @@ import pytest -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.enums.user_role import UserRole @@ -52,8 +52,8 @@ def setup_router_db(session): db.add(License(**TEST_INACTIVE_LICENSE)) db.add(License(**EXTRA_LICENSE)) db.add(Contributor(**EXTRA_CONTRIBUTOR)) - db.add(ClinicalControl(**TEST_CLINVAR_CONTROL)) - db.add(ClinicalControl(**TEST_GENERIC_CLINICAL_CONTROL)) + db.add(ClinvarControl(**TEST_CLINVAR_CONTROL)) + db.add(ClinvarControl(**TEST_GENERIC_CLINICAL_CONTROL)) db.add(GnomADVariant(**TEST_GNOMAD_VARIANT)) db.bulk_save_objects([ControlledKeyword(**keyword_obj) for keyword_obj in TEST_DB_KEYWORDS]) db.commit() diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index f29842158..4569d08f7 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,11 +10,11 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data -from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.clinvar import generate_clinvar_versions +from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -43,7 +43,7 @@ async def test_refresh_clinvar_controls_e2e( arq_redis, arq_worker, standalone_worker_context, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): @@ -65,8 +65,9 @@ async def test_refresh_clinvar_controls_e2e( await arq_worker.async_run() await arq_worker.run_check() - # The variant should be present in the 2025-01 archive. - clinical_controls = session.scalars(select(ClinicalControl)).all() + # Verify that clinical controls were added successfully — one row per ClinVar version + # that contains the variant, so there may be more than one. + clinical_controls = session.scalars(select(ClinvarControl)).all() assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) @@ -101,7 +102,7 @@ async def test_latest_clinvar_archive_has_expected_schema(self): CLINVAR_FIELDS_TO_KEEP is present. Fails fast if ClinVar renames or removes a column before a deploy reaches production. """ - year, month = generate_clinvar_versions()[-1] + year, month = _generate_clinvar_versions()[-1] data = await fetch_clinvar_variant_data(month, year) assert len(data) > 0, "ClinVar archive returned no records" diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 1ac616ffc..6b5011236 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -3,19 +3,18 @@ import pytest import requests -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, FailureCategory, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - pytest.importorskip("arq") from unittest.mock import patch +from sqlalchemy import select + from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory, JobStatus +from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -26,6 +25,7 @@ "GeneSymbol": "TEST", "ClinicalSignificance": "benign", "ReviewStatus": "reviewed by expert panel", + "VariationID": "987654", }, } @@ -33,32 +33,29 @@ @pytest.mark.unit @pytest.mark.asyncio class TestRefreshClinvarControlsUnit: - """Tests for the refresh_clinvar_controls job function.""" + """Unit tests for the allele-model refresh_clinvar_controls job.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" + """Pin _generate_clinvar_versions to a single version so clinvar_version == '01_2026'.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_skips_version_on_fetch_failure( + async def test_no_alleles_with_caids( self, mock_worker_ctx, session, + with_populated_domain_data, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, ): - """Test that a fetch failure for a version is logged and skipped, not propagated.""" - - async def awaitable_exception(*args, **kwargs): - raise Exception("Network error") - + """No current alleles carry a CAID -> the job succeeds with nothing to do.""" with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=awaitable_exception, + return_value=MOCK_CLINVAR_DATA, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -68,61 +65,26 @@ async def awaitable_exception(*args, **kwargs): assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarControl)).all() == [] + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.query(VariantAnnotationStatus).all() == [] - async def test_refresh_clinvar_controls_no_mapped_variants( + async def test_fetch_failure_skips_version( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when there are no mapped variants.""" - - with patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value={}, - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + """A version whose TSV fetch fails is logged and skipped, not propagated.""" - async def test_refresh_clinvar_controls_no_variants_have_caids( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - ): - """Test that the job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.2G>A", - hgvs_pro="NP_000000.1:p.Val2Ile", - data={"hgvs_c": "NM_000000.1:c.2G>A", "hgvs_p": "NP_000000.1:p.Val2Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + async def boom(*args, **kwargs): + raise Exception("Network error") with patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + side_effect=boom, ): result = await refresh_clinvar_controls( mock_worker_ctx, @@ -130,29 +92,21 @@ async def test_refresh_clinvar_controls_no_variants_have_caids( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert result.data["versions_completed"] == 0 + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - async def test_refresh_clinvar_controls_variants_are_multivariants( + async def test_multi_variant_caid_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job completes successfully when all variants are multi-variant CAIDs.""" - # Update the mapped variant to have a multi-variant CAID - mapped_variant = session.query(MappedVariant).first() - mapped_variant.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" + """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" + variant, allele = setup_sample_alleles_with_caid + allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() with patch( @@ -165,37 +119,29 @@ async def test_refresh_clinvar_controls_variants_are_multivariants( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.annotation_type == AnnotationType.CLINVAR_CONTROL + assert vas.failure_category == AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER - async def test_refresh_clinvar_controls_clingen_api_failure( + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles ClinGen API failures gracefully.""" + """ClinGen returns no ClinVar allele id for the CAID -> SKIPPED/NO_LINKED_ALLELE, no link.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to raise an exception with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), + return_value=None, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -208,40 +154,31 @@ async def test_refresh_clinvar_controls_clingen_api_failure( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - - # Verify an annotation status was created for the variant due to ClinGen API failure - mapped_variant = session.query(MappedVariant).first() - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.failure_category == AnnotationFailureCategory.NO_LINKED_ALLELE - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_clinvar_data_not_found_skipped( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no associated ClinVar Allele ID gracefully.""" + """The resolved ClinVar allele id is absent from the version's TSV -> SKIPPED, no link.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return None with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + return_value="VCV000000123", ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value={"VCV000000999": {"GeneSymbol": "X", "ClinicalSignificance": "y", "ReviewStatus": "z"}}, ), ): result = await refresh_clinvar_controls( @@ -250,48 +187,31 @@ async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SKIPPED + assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message - - async def test_refresh_clinvar_controls_no_clinvar_data_found( + async def test_clingen_api_failure_fails_when_total( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles no ClinVar data found for the associated ClinVar Allele ID.""" - - # TSV data with a different allele ID than the one being looked up - non_matching_clinvar_data = { - "VCV000000001": { - "GeneSymbol": "TEST", - "ClinicalSignificance": "benign", - "ReviewStatus": "reviewed by expert panel", - }, - } + """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" + variant, _ = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=requests.exceptions.RequestException("ClinGen API error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=non_matching_clinvar_data, + return_value=MOCK_CLINVAR_DATA, ), ): result = await refresh_clinvar_controls( @@ -300,31 +220,23 @@ async def test_refresh_clinvar_controls_no_clinvar_data_found( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - mapped_variant = session.query(MappedVariant).first() - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message + assert result.status == JobStatus.FAILED + assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.FAILED + assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_API_ERROR - async def test_refresh_clinvar_controls_successful_annotation_existing_control( + async def test_successful_link_new_control( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data.""" + """A resolved + present CAID creates a ClinvarControl, a live link, and a SUCCESS VAS.""" + variant, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -341,54 +253,48 @@ async def test_refresh_clinvar_controls_successful_annotation_existing_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - mapped_variant = session.query(MappedVariant).first() - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - async def test_refresh_clinvar_controls_successful_annotation_new_control( + assert result.data["created_link_count"] == 1 + + control = session.scalars(select(ClinvarControl)).one() + assert control.db_identifier == "VCV000000123" + assert control.db_version == "01_2026" + assert control.clinical_significance == "benign" + # AlleleID stays in db_identifier; the canonical VariationID is captured alongside it. + assert control.clinvar_variation_id == "987654" + + links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id, ClinvarAlleleLink.current) + ).all() + assert len(links) == 1 + assert links[0].clinvar_control_id == control.id + + vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() + assert vas.status == AnnotationStatus.SUCCESS + assert vas.annotation_metadata["action"] == "created" + + async def test_links_rt_derived_allele_but_annotates_only_authoritative( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, + setup_rt_derived_allele_with_caid, ): - """Test that the job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant and mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.3C>T", - hgvs_pro="NP_000000.1:p.Ala3Val", - data={"hgvs_c": "NM_000000.1:c.3C>T", "hgvs_p": "NP_000000.1:p.Ala3Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA124", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() + """ClinVar linkage covers the full allele set: the RT-derived allele carries the matching CAID + and is linked, while per-variant VAS is written only for the authoritative link (the bandaid). + """ + variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + rt_caid = rt_allele.clingen_allele_id + + async def resolve(caid): + # Only the RT-derived allele's CAID resolves to a ClinVar id present in the TSV. + return "VCV000000123" if caid == rt_caid else None - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", + side_effect=resolve, ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", @@ -401,1105 +307,276 @@ async def test_refresh_clinvar_controls_successful_annotation_new_control( JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() + # The RT-derived allele IS linked — the core fix. + rt_links = session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == rt_allele.id, ClinvarAlleleLink.current) + ).all() + assert len(rt_links) == 1 + # The authoritative allele's CAID had no ClinVar match, so it is not linked. + assert ( + session.scalars( + select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == authoritative_allele.id) + ).all() + == [] ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - async def test_refresh_clinvar_controls_idempotent_run( + # VAS is written only for the authoritative link: exactly one row, keyed to the variant. + statuses = session.query(VariantAnnotationStatus).all() + assert len(statuses) == 1 + assert statuses[0].variant_id == variant.id + assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + + async def test_idempotent_rerun_skips_and_does_not_duplicate( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that running the job multiple times does not create duplicate annotation statuses.""" + """A second run finds the allele already linked at the version, skips the resolution, reports + preexisting, and creates neither a duplicate control nor a duplicate link.""" + variant, allele = setup_sample_alleles_with_caid - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], + return_value=MOCK_CLINVAR_DATA, ), ): - # First run - result1 = await refresh_clinvar_controls( + first = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - session.commit() + assert resolve_spy.await_count == 1 - # Second run - result2 = await refresh_clinvar_controls( + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) + # Second run skipped the resolution entirely (version-keyed skip). + assert resolve_spy.await_count == 1 - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - async def test_refresh_clinvar_controls_partial_failure( + assert first.data["created_link_count"] == 1 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + + # One control, one live link — no churn. + assert len(session.scalars(select(ClinvarControl)).all()) == 1 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None + + # Per-version VAS: two rows for the variant, only one current. + statuses = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).all() + assert len(statuses) == 2 + assert len([s for s in statuses if s.current]) == 1 + + async def test_release_reresolution_supersedes_newest_wins( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.4G>C", - hgvs_pro="NP_000000.1:p.Gly4Ala", - data={"hgvs_c": "NM_000000.1:c.4G>C", "hgvs_p": "NP_000000.1:p.Gly4Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA125", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) + """Defensive guard: if an allele re-resolves to a *different* control within the same release + (should never happen — archival data is immutable), the old live link is superseded + newest-wins, leaving exactly one live link per (allele, version).""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA125": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" + tsv = { + "VCV000000123": { + "GeneSymbol": "A", + "ClinicalSignificance": "benign", + "ReviewStatus": "ok", + "VariationID": "111", + }, + "VCV000000999": { + "GeneSymbol": "B", + "ClinicalSignificance": "pathogenic", + "ReviewStatus": "ok", + "VariationID": "222", + }, + } with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, + side_effect=["VCV000000123", "VCV000000999"], ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + return_value=tsv, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - async def test_total_api_failure_returns_failed( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that the job returns FAILED when all ClinVar lookups fail.""" - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=requests.exceptions.RequestException("ClinGen API error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( + session.commit() + await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE + control_a = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000123")).one() + control_b = session.scalars(select(ClinvarControl).where(ClinvarControl.db_identifier == "VCV000000999")).one() - async def test_upsert_does_not_create_duplicate_control_when_row_already_exists( + all_links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + live_links = [link for link in all_links if link.valid_to is None] + + # Exactly one live link per (allele, version) — the new control — and the old one is retired. + assert len(live_links) == 1 + assert live_links[0].clinvar_control_id == control_b.id + retired = [link for link in all_links if link.clinvar_control_id == control_a.id] + assert len(retired) == 1 + assert retired[0].valid_to is not None + # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. + assert retired[0].valid_to == live_links[0].valid_from + + async def test_force_reresolves_without_duplicating_link( self, mock_worker_ctx, session, with_refresh_clinvar_controls_job, sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, + setup_sample_alleles_with_caid, ): - """Test that the upsert handles a pre-existing ClinicalControl row without creating a duplicate. - - This covers the concurrent-job race condition where two refresh_clinvar_controls - jobs run simultaneously for different score sets and both try to insert a - ClinicalControl for the same (db_name, db_identifier, db_version). The second - job's upsert must hit ON CONFLICT DO UPDATE rather than inserting a second row. - """ - # Simulate the state left by a concurrent job: the ClinicalControl row - # for this identifier/version already exists in the DB with stale data. - pre_existing_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - db_version="01_2026", - gene_symbol="OLD_SYMBOL", - clinical_significance="likely pathogenic", - clinical_review_status="criteria provided, single submitter", - ) - session.add(pre_existing_control) + """force bypasses the version-keyed skip and re-resolves, but the get-or-create link write + does not duplicate an existing live link.""" + variant, allele = setup_sample_alleles_with_caid + sample_refresh_clinvar_controls_job_run.job_params = { + **sample_refresh_clinvar_controls_job_run.job_params, + "force": True, + } session.commit() with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", return_value="VCV000000123", - ), + ) as resolve_spy, patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls( + await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + session.commit() + second = await refresh_clinvar_controls( mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id, JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), ) - assert result.status == JobStatus.SUCCEEDED - - # Only one row should exist — the upsert must not have inserted a second one. - controls = session.query(ClinicalControl).filter_by(db_identifier="VCV000000123", db_version="01_2026").all() - assert len(controls) == 1 - - # The upsert should have updated the stale data from the concurrent job. - session.refresh(controls[0]) - assert controls[0].gene_symbol == "TEST" - assert controls[0].clinical_significance == "benign" - assert controls[0].clinical_review_status == "reviewed by expert panel" + # force re-resolved on the second run despite an existing link. + assert resolve_spy.await_count == 2 + assert second.data["preexisting_link_count"] == 1 + assert second.data["created_link_count"] == 0 + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.allele_id == allele.id)).all() + assert len(links) == 1 + assert links[0].valid_to is None @pytest.mark.integration @pytest.mark.asyncio -class TestRefreshClinvarControlsIntegration: - """Integration tests for the refresh_clinvar_controls job function.""" +class TestRefreshClinvarControlsArqContext: + """End-to-end tests for refresh_clinvar_controls within an ARQ worker context.""" @pytest.fixture(autouse=True) def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=[(2026, 1)], ): yield - async def test_refresh_clinvar_controls_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when there are no mapped variants.""" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify no controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_variants_with_caid( + async def test_arq_context_successful_link( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job completes successfully when no variants have CAIDs.""" - # Add a variant without a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-no-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.5T>A", - hgvs_pro="NP_000000.1:p.Leu5Gln", - data={"hgvs_c": "NM_000000.1:c.5T>A", "hgvs_p": "NP_000000.1:p.Leu5Gln"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - + """The job links an allele and records a SUCCESS annotation under an ARQ worker.""" with ( patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant without a CAID - variant_no_caid = ( - session.query(VariantAnnotationStatus).filter(VariantAnnotationStatus.variant_id == variant.id).one() - ) - assert variant_no_caid.status == AnnotationStatus.SKIPPED - assert variant_no_caid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert variant_no_caid.error_message == "Mapped variant does not have an associated ClinGen allele ID." - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controlsvariants_are_multivariants( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job completes successfully when all variants are multi-variant CAIDs.""" - # Add a variant with a multi-variant CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-multicid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.6A>G", - hgvs_pro="NP_000000.1:p.Thr6Ala", - data={"hgvs_c": "NM_000000.1:c.6A>G", "hgvs_p": "NP_000000.1:p.Thr6Ala"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA-MULTI-003,CA-MULTI-004", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - with ( patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify an annotation status was created for the multi-variant CAID - variant_with_multicid = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_with_multicid.status == AnnotationStatus.SKIPPED - assert variant_with_multicid.annotation_type == AnnotationType.CLINVAR_CONTROL - assert ( - variant_with_multicid.error_message - == "Multi-variant ClinGen allele IDs cannot be associated with ClinVar data." - ) + assert len(session.scalars(select(ClinvarControl)).all()) >= 1 + assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + statuses = session.query(VariantAnnotationStatus).all() + assert len(statuses) == 1 + assert statuses[0].status == AnnotationStatus.SUCCESS + assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - # Verify job run status is marked as completed session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - async def test_refresh_clinvar_controls_no_associated_clinvar_allele_id( + async def test_arq_context_exception_handling( self, + arq_redis, + arq_worker, session, with_populated_domain_data, with_refresh_clinvar_controls_job, - mock_worker_ctx, sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, ): - """Integration test: job handles no associated ClinVar Allele ID gracefully.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.7C>A", - hgvs_pro="NP_000000.1:p.Ser7Tyr", - data={"hgvs_c": "NM_000000.1:c.7C>A", "hgvs_p": "NP_000000.1:p.Ser7Tyr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA126", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return None + """An unexpected error during resolution is caught by the decorators and the job errors.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value=None, + side_effect=ValueError("Unexpected error"), ), patch( "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", return_value=MOCK_CLINVAR_DATA, ), + patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_slack, ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no associated ClinVar Allele ID - variant_no_clinvar_allele = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_allele.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_allele.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar allele ID found for ClinGen allele ID" in variant_no_clinvar_allele.error_message + await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) + await arq_worker.async_run() + await arq_worker.run_check() - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 + mock_slack.assert_called_once() + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.query(VariantAnnotationStatus).all() == [] - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_no_clinvar_data( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job handles no ClinVar data found for the associated ClinVar Allele ID.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.8G>T", - hgvs_pro="NP_000000.1:p.Val8Phe", - data={"hgvs_c": "NM_000000.1:c.8G>T", "hgvs_p": "NP_000000.1:p.Val8Phe"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA127", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000001", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant due to no ClinVar data found - variant_no_clinvar_data = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert variant_no_clinvar_data.status == AnnotationStatus.SKIPPED - assert variant_no_clinvar_data.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "No ClinVar data found for ClinVar allele ID" in variant_no_clinvar_data.error_message - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_existing_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.9A>C", - hgvs_pro="NP_000000.1:p.Lys9Thr", - data={"hgvs_c": "NM_000000.1:c.9A>C", "hgvs_p": "NP_000000.1:p.Lys9Thr"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA128", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - clinical_control = ClinicalControl( - db_name="ClinVar", - db_identifier="VCV000000123", - clinical_significance="likely pathogenic", - gene_symbol="TEST", - clinical_review_status="criteria provided, single submitter", - db_version="01_2026", - ) - session.add(clinical_control) - session.commit() - - mapped_variant.clinical_controls.append(clinical_control) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was updated - session.refresh(clinical_control) - assert clinical_control.clinical_significance == "benign" - assert clinical_control.clinical_review_status == "reviewed by expert panel" - assert mapped_variant in clinical_control.mapped_variants - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_new_control( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - ): - """Integration test: job successfully annotates a variant with ClinVar control data when no prior status exists.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.10C>G", - hgvs_pro="NP_000000.1:p.Pro10Arg", - data={"hgvs_c": "NM_000000.1:c.10C>G", "hgvs_p": "NP_000000.1:p.Pro10Arg"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA129", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_successful_annotation_pipeline_context( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_pipeline, - sample_refresh_clinvar_controls_job_in_pipeline, - ): - """Integration test: job successfully annotates a variant with ClinVar control data in a pipeline context.""" - # Add a variant with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_in_pipeline.job_params["score_set_id"]) - variant = Variant( - urn="urn:variant:integration-test-variant-with-caid", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.12G>A", - hgvs_pro="NP_000000.1:p.Met12Ile", - data={"hgvs_c": "NM_000000.1:c.12G>A", "hgvs_p": "NP_000000.1:p.Met12Ile"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_in_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify an annotation status was created for the variant with successful annotation - annotated_variant = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant.variant_id) - .one() - ) - assert annotated_variant.status == AnnotationStatus.SUCCESS - assert annotated_variant.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant.error_message is None - - # Verify the clinical control was added - clinical_control = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant)).one() - ) - assert clinical_control.db_identifier == "VCV000000123" - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_in_pipeline) - assert sample_refresh_clinvar_controls_job_in_pipeline.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - session.refresh(sample_refresh_clinvar_controls_pipeline) - assert sample_refresh_clinvar_controls_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_idempotent_run( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: running the job multiple times does not create duplicate annotation statuses.""" - - # Mock the get_associated_clinvar_allele_id function to return a ClinVar Allele ID - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - side_effect=[MOCK_CLINVAR_DATA, MOCK_CLINVAR_DATA], - ), - ): - # First run - result1 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - session.commit() - # reset the job run status to pending for the second run - sample_refresh_clinvar_controls_job_run.status = JobStatus.PENDING - session.commit() - - # Second run - result2 = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result1, JobExecutionOutcome) - assert result1.status == JobStatus.SUCCEEDED - assert isinstance(result2, JobExecutionOutcome) - assert result2.status == JobStatus.SUCCEEDED - - # Verify only one clinical control annotation exists for the variant - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 1 - - # Verify two annotated variants exist but both reflect the same successful annotation, and only - # one is current - annotated_variants = session.query(VariantAnnotationStatus).all() - assert len(annotated_variants) == 2 - statuses = [av.status for av in annotated_variants] - assert statuses.count(AnnotationStatus.SUCCESS) == 2 - current_statuses = [av for av in annotated_variants if av.current] - assert len(current_statuses) == 1 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_partial_failure( - self, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles partial failures gracefully.""" - - variant1, mapped_variant1 = setup_sample_variants_with_caid - # Add an additional mapped variant to the database with a CAID - score_set = session.get(ScoreSet, sample_refresh_clinvar_controls_job_run.job_params["score_set_id"]) - variant2 = Variant( - urn="urn:variant:integration-test-variant-with-caid-2", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.11G>C", - hgvs_pro="NP_000000.1:p.Gly11Ala", - data={"hgvs_c": "NM_000000.1:c.11G>C", "hgvs_p": "NP_000000.1:p.Gly11Ala"}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA130", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Mock the get_associated_clinvar_allele_id function to raise an exception for the first call - def side_effect_get_associated_clinvar_allele_id(clingen_allele_id): - if clingen_allele_id == "CA130": - raise requests.exceptions.RequestException("ClinGen API error") - return "VCV000000123" - - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=side_effect_get_associated_clinvar_allele_id, - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls(mock_worker_ctx, sample_refresh_clinvar_controls_job_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify annotation statuses for both variants - variant_with_api_failure = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant2.variant_id) - .one() - ) - assert variant_with_api_failure.status == AnnotationStatus.FAILED - assert variant_with_api_failure.annotation_type == AnnotationType.CLINVAR_CONTROL - assert "Failed to retrieve ClinVar allele ID from ClinGen API" in variant_with_api_failure.error_message - - annotated_variant2 = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == mapped_variant1.variant_id) - .one() - ) - assert annotated_variant2.status == AnnotationStatus.SUCCESS - assert annotated_variant2.annotation_type == AnnotationType.CLINVAR_CONTROL - assert annotated_variant2.error_message is None - - # Verify a clinical control was added for the successfully annotated variant and not the unsuccessful one - clinical_control1 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant1)).one() - ) - assert clinical_control1.db_identifier == "VCV000000123" - - clinical_control2 = ( - session.query(ClinicalControl).filter(ClinicalControl.mapped_variants.contains(mapped_variant2)).all() - ) - assert len(clinical_control2) == 0 - - # Verify job run status is marked as completed - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_propagates_exceptions_to_decorator( - self, - mock_worker_ctx, - session, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Test that unexpected exceptions are propagated.""" - - # Mock the get_associated_clinvar_allele_id function to raise an unexpected exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - result = await refresh_clinvar_controls( - mock_worker_ctx, - sample_refresh_clinvar_controls_job_run.id, - JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestRefreshClinvarControlsArqContext: - """Tests for running the refresh_clinvar_controls job function within an ARQ worker context.""" - - @pytest.fixture(autouse=True) - def _mock_clinvar_versions(self): - """Mock generate_clinvar_versions to return a single version for testing.""" - with patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", - return_value=[(2026, 1)], - ): - yield - - async def test_refresh_clinvar_controls_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - async def test_refresh_clinvar_controls_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job completes successfully within an ARQ worker context in a pipeline context.""" - - # Patch external service calls - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - return_value="VCV000000123", - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify that clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) > 0 - - # Verify annotation status was created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == AnnotationStatus.SUCCESS - assert annotation_statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL - - # Verify that the job completed successfully - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED - - # Verify the pipeline is marked as completed - pass - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) - session.refresh(sample_refresh_clinvar_controls_job_run) - assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - async def test_refresh_clinvar_controls_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_refresh_clinvar_controls_job, - sample_refresh_clinvar_controls_job_run, - setup_sample_variants_with_caid, - ): - """Integration test: job handles exceptions properly within an ARQ worker context in a pipeline context.""" - # Patch external service calls to raise an exception - with ( - patch( - "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", - side_effect=ValueError("Unexpected error"), - ), - patch( - "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", - return_value=MOCK_CLINVAR_DATA, - ), - ): - await arq_redis.enqueue_job("refresh_clinvar_controls", sample_refresh_clinvar_controls_job_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify no clinical controls were added - clinical_controls = session.query(ClinicalControl).all() - assert len(clinical_controls) == 0 - - # Verify job run status is marked as errored (unhandled exception caught by decorator) session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED - - # Verify the pipeline is marked as failed - pass From 3d9b5fa836d39ad7c48dc31f53968ec92e8cab40 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 23 Jun 2026 09:26:05 -0700 Subject: [PATCH 28/93] refactor(annotation): retire the HGVS and variant-translation jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 4 & 5 of the #742 migration. Both jobs are redundant under the allele model: Allele.hgvs_g/c/p are populated by the mapping job, and the reverse-translation equivalence space (genomic/coding/protein alleles linked per MappingRecord) replaces the ClinGen PA<->CA translation table. - Remove populate_hgvs_for_score_set and populate_variant_translations_for_score_set from the pipeline DAG (both were leaf nodes — no dependency edges to repair), the worker registry (BACKGROUND_FUNCTIONS + STANDALONE_JOB_DEFINITIONS), and the external_services package exports. - Delete the two job modules and the now-orphaned ClinGen HGVS helpers (extract_hgvs_from_ca_allele_data / extract_hgvs_from_pa_allele_data), used only by the HGVS job. - Keep lib/variant_translations.py and the variant_translations table/model, marked FROZEN (serving-only) — they back old-model serving and are dropped at read-cutover. - Delete the obsolete job tests and their conftest fixtures. --- src/mavedb/lib/clingen/allele_registry.py | 74 -- src/mavedb/lib/variant_translations.py | 6 + src/mavedb/lib/workflow/definitions.py | 20 - src/mavedb/models/variant_translation.py | 4 + .../worker/jobs/external_services/__init__.py | 6 - .../worker/jobs/external_services/hgvs.py | 298 ------- .../external_services/variant_translation.py | 371 --------- src/mavedb/worker/jobs/registry.py | 18 - tests/worker/jobs/conftest.py | 192 ----- .../external_services/network/test_hgvs.py | 54 -- .../network/test_variant_translations.py | 56 -- .../jobs/external_services/test_hgvs.py | 544 ------------- .../test_variant_translation.py | 770 ------------------ 13 files changed, 10 insertions(+), 2403 deletions(-) delete mode 100644 src/mavedb/worker/jobs/external_services/hgvs.py delete mode 100644 src/mavedb/worker/jobs/external_services/variant_translation.py delete mode 100644 tests/worker/jobs/external_services/network/test_hgvs.py delete mode 100644 tests/worker/jobs/external_services/network/test_variant_translations.py delete mode 100644 tests/worker/jobs/external_services/test_hgvs.py delete mode 100644 tests/worker/jobs/external_services/test_variant_translation.py diff --git a/src/mavedb/lib/clingen/allele_registry.py b/src/mavedb/lib/clingen/allele_registry.py index b773e6891..b28a184d1 100644 --- a/src/mavedb/lib/clingen/allele_registry.py +++ b/src/mavedb/lib/clingen/allele_registry.py @@ -136,80 +136,6 @@ async def get_associated_clinvar_allele_id(clingen_allele_id: str) -> str: return "" -def extract_hgvs_from_ca_allele_data( - data: dict, - target_is_coding: bool, - transcript_accession: Optional[str], -) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a CA (canonical allele) ID. - - Parses the ClinGen API response to find GRCh38 genomic HGVS, coding HGVS - matching the target transcript (or MANE fallback), and protein HGVS. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - target_is_coding: Whether the score set target is protein-coding. - transcript_accession: Specific transcript accession to match, or None to use MANE. - - Returns: - Tuple of (hgvs_g, hgvs_c, hgvs_p), any of which may be None. - """ - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - if data.get("genomicAlleles"): - for allele in data["genomicAlleles"]: - if allele.get("referenceGenome") == "GRCh38" and allele.get("hgvs"): - hgvs_g = allele["hgvs"][0] - break - - if target_is_coding and data.get("transcriptAlleles"): - if transcript_accession: - for allele in data["transcriptAlleles"]: - if allele.get("hgvs"): - for hgvs_string in allele["hgvs"]: - hgvs_reference_sequence = hgvs_string.split(":")[0] - if transcript_accession == hgvs_reference_sequence: - hgvs_c = hgvs_string - break - if hgvs_c: - if allele.get("proteinEffect"): - hgvs_p = allele["proteinEffect"].get("hgvs") - break - else: - # No transcript specified; use MANE if available - for allele in data["transcriptAlleles"]: - if allele.get("MANE"): - hgvs_c = allele["MANE"].get("nucleotide", {}).get("RefSeq", {}).get("hgvs") - hgvs_p = allele["MANE"].get("protein", {}).get("RefSeq", {}).get("hgvs") - break - - return hgvs_g, hgvs_c, hgvs_p - - -def extract_hgvs_from_pa_allele_data(data: dict) -> tuple[Optional[str], Optional[str], Optional[str]]: - """Extract HGVS strings from ClinGen allele data for a PA (protein allele) ID. - - For PA alleles, only hgvs_p is extracted from aminoAcidAlleles. - - Args: - data: Parsed JSON response from the ClinGen Allele Registry API. - - Returns: - Tuple of (None, None, hgvs_p), where hgvs_p may be None. - """ - hgvs_p: Optional[str] = None - - if data.get("aminoAcidAlleles"): - for allele in data["aminoAcidAlleles"]: - if allele.get("hgvs"): - hgvs_p = allele["hgvs"][0] - break - - return None, None, hgvs_p - - def expand_allele_ids(clingen_allele_ids: list[Optional[str]]) -> set[str]: """Expand comma-separated multi-variant ClinGen allele IDs into individual IDs. diff --git a/src/mavedb/lib/variant_translations.py b/src/mavedb/lib/variant_translations.py index 701cc17d1..31d89e6df 100644 --- a/src/mavedb/lib/variant_translations.py +++ b/src/mavedb/lib/variant_translations.py @@ -3,6 +3,12 @@ This module provides database operations for the variant_translations table, which stores relationships between protein allele (PA) and nucleotide allele (CA) ClinGen IDs. + +FROZEN (serving-only). The populate_variant_translations_for_score_set job that wrote this table was +retired in the #742 migration: the reverse-translation allele equivalence space (genomic/coding/protein +VRS alleles per variant, linked via MappingRecordAllele with HGVS on Allele) now covers PA<->CA +relationships without querying ClinGen. These helpers and the variant_translations table remain only to +serve existing old-model data; they are never written for new score sets and are dropped at read-cutover. """ from typing import cast diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 3d3b70511..93b4b6de1 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -96,16 +96,6 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_hgvs_for_score_set", - "function": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, { "key": "populate_vep_for_score_set", "function": "populate_vep_for_score_set", @@ -116,16 +106,6 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_variant_translations_for_score_set", - "function": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], - }, ] diff --git a/src/mavedb/models/variant_translation.py b/src/mavedb/models/variant_translation.py index a50d36ce2..36aa0dfdf 100644 --- a/src/mavedb/models/variant_translation.py +++ b/src/mavedb/models/variant_translation.py @@ -6,6 +6,10 @@ class VariantTranslation(Base): + """FROZEN (serving-only). Written by the retired populate_variant_translations_for_score_set job; + superseded by the reverse-translation allele equivalence space. Read for existing old-model data, + never written for new score sets, dropped at read-cutover. See lib/variant_translations.py.""" + __tablename__ = "variant_translations" aa_clingen_id = Column(String, nullable=False, primary_key=True) diff --git a/src/mavedb/worker/jobs/external_services/__init__.py b/src/mavedb/worker/jobs/external_services/__init__.py index 4537c0edd..78af92516 100644 --- a/src/mavedb/worker/jobs/external_services/__init__.py +++ b/src/mavedb/worker/jobs/external_services/__init__.py @@ -5,8 +5,6 @@ - ClinGen cache pre-warming to prevent stampede on downstream annotation jobs - UniProt for protein sequence annotation and ID mapping - gnomAD for population frequency and genomic context data -- HGVS for standardized variant nomenclature population -- Variant Translation for PA<->CA allele relationship mapping - VEP for functional consequence annotation """ @@ -18,12 +16,10 @@ from .clingen_cache import warm_clingen_cache from .clinvar import refresh_clinvar_controls from .gnomad import link_gnomad_variants -from .hgvs import populate_hgvs_for_score_set from .uniprot import ( poll_uniprot_mapping_jobs_for_score_set, submit_uniprot_mapping_jobs_for_score_set, ) -from .variant_translation import populate_variant_translations_for_score_set from .vep import populate_vep_for_score_set __all__ = [ @@ -32,8 +28,6 @@ "warm_clingen_cache", "refresh_clinvar_controls", "link_gnomad_variants", - "populate_hgvs_for_score_set", - "populate_variant_translations_for_score_set", "poll_uniprot_mapping_jobs_for_score_set", "submit_uniprot_mapping_jobs_for_score_set", "populate_vep_for_score_set", diff --git a/src/mavedb/worker/jobs/external_services/hgvs.py b/src/mavedb/worker/jobs/external_services/hgvs.py deleted file mode 100644 index 0b4687398..000000000 --- a/src/mavedb/worker/jobs/external_services/hgvs.py +++ /dev/null @@ -1,298 +0,0 @@ -"""ClinGen allele HGVS population jobs for mapped variant annotation. - -This module populates mapped variants with HGVS representations (genomic, coding, -protein) by querying the ClinGen Allele Registry. It uses ClinGen allele IDs -(CAIDs) already associated with mapped variants to look up standardized HGVS -nomenclature at different levels (hgvs_g, hgvs_c, hgvs_p), plus the assay-level -HGVS derived from post-mapped VRS data. -""" - -import logging -from typing import Optional - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - extract_hgvs_from_ca_allele_data, - extract_hgvs_from_pa_allele_data, - get_clingen_allele_data, -) -from mavedb.lib.slack import log_and_send_slack_message -from mavedb.lib.target_genes import get_target_coding_info -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_hgvs_for_score_set(ctx: dict, job_id: int, job_manager: JobManager) -> JobExecutionOutcome: - """Populate mapped variants with HGVS representations for a score set. - - Queries the ClinGen Allele Registry using existing ClinGen allele IDs to populate - standardized HGVS nomenclature (genomic, coding, protein) on mapped variants. - Also extracts the assay-level HGVS from post-mapped VRS data. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - - Args: - ctx: Worker context containing DB and Redis connections. - job_id: The ID of the job run. - job_manager: Manager for job lifecycle and DB operations. - - Side Effects: - - Updates MappedVariant records with hgvs_assay_level, hgvs_g, hgvs_c, hgvs_p. - - Creates AnnotationStatus records for each processed variant. - - Returns: - JobExecutionOutcome indicating success, failure, or skip. - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - # Fetch required resources based on param inputs. Safely ignore mypy warnings here, as they were checked above. - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - # Setup initial context and progress - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_hgvs_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting mapped HGVS population.") - logger.info(msg="Started mapped HGVS population", extra=job_manager.logging_context()) - - # Determine target info; multi-target score sets are not yet supported - try: - target_is_coding, transcript_accession = get_target_coding_info(score_set) - except NotImplementedError: - logger.warning( - msg="Multi-target score sets not supported for HGVS population. Skipping.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.skipped(data={"reason": "Multi-target score sets not supported"}) - - job_manager.save_to_context({"target_is_coding": target_is_coding, "transcript_accession": transcript_accession}) - logger.info( - msg=f"Target info resolved: coding={target_is_coding}, transcript={transcript_accession}", - extra=job_manager.logging_context(), - ) - - # Fetch current mapped variants for the score set - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant) - .join(Variant) - .join(ScoreSet) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - total_variants = len(variant_rows) - job_manager.save_to_context({"total_variants": total_variants}) - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set. Skipping HGVS population.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"populated_count": 0, "skipped_count": 0, "failed_count": 0}) - - job_manager.update_progress(5, 100, f"Processing {total_variants} mapped variants for HGVS population.") - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - populated_count = 0 - skipped_count = 0 - failed_count = 0 - - for index, (variant_id, mapped_variant) in enumerate(variant_rows): - # Periodic progress updates - if total_variants > 0 and index % max(total_variants // 20, 1) == 0: - progress = 5 + int((index / total_variants) * 90) - job_manager.update_progress(progress, 100, f"Processing HGVS for variant {index + 1}/{total_variants}.") - logger.info( - "Processing variant %s/%s: variant_id=%s", - index + 1, - total_variants, - variant_id, - extra=job_manager.logging_context(), - ) - - hgvs_g: Optional[str] = None - hgvs_c: Optional[str] = None - hgvs_p: Optional[str] = None - - clingen_id = mapped_variant.clingen_allele_id - - job_manager.save_to_context( - { - "mapped_variant_id": mapped_variant.id, - "clingen_allele_id": clingen_id, - "progress_index": index, - } - ) - - if not clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, - annotation_data={ - "error_message": "No ClinGen allele ID available for ClinGen HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: no ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Skip multi-variant allele IDs (comma-separated) - if "," in clingen_id: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": "Multi-variant ClinGen allele IDs not supported for HGVS lookup.", - }, - current=True, - ) - logger.debug( - "Skipping variant %s: multi-variant ClinGen allele ID.", - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Query ClinGen API for allele data - try: - allele_data = await get_clingen_allele_data(clingen_id) - except requests.exceptions.RequestException as exc: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"Failed to fetch ClinGen allele data: {str(exc)}", - }, - current=True, - ) - logger.error( - "ClinGen API request failed for allele %s.", - clingen_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed_count += 1 - continue - - if allele_data is None: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - annotation_data={ - "error_message": f"ClinGen allele {clingen_id} not found in the registry.", - }, - current=True, - ) - logger.debug( - "ClinGen allele %s not found in registry. Skipping variant %s.", - clingen_id, - variant_id, - extra=job_manager.logging_context(), - ) - skipped_count += 1 - continue - - # Extract HGVS based on allele type - if clingen_id.startswith("CA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_ca_allele_data( - allele_data, target_is_coding, transcript_accession - ) - elif clingen_id.startswith("PA"): - hgvs_g, hgvs_c, hgvs_p = extract_hgvs_from_pa_allele_data(allele_data) - - # Update mapped variant - mapped_variant.hgvs_g = hgvs_g - mapped_variant.hgvs_c = hgvs_c - mapped_variant.hgvs_p = hgvs_p - job_manager.db.add(mapped_variant) - - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.MAPPED_HGVS, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "hgvs_g": hgvs_g, - "hgvs_c": hgvs_c, - "hgvs_p": hgvs_p, - }, - }, - current=True, - ) - populated_count += 1 - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) - logger.info( - msg=f"Completed mapped HGVS population: {populated_count} populated, {skipped_count} skipped, {failed_count} failed.", - extra=job_manager.logging_context(), - ) - - if failed_count > 0 and populated_count == 0: - log_and_send_slack_message( - f"All {failed_count} variants failed HGVS population for score set {score_set.urn}. Possible ClinGen API outage.", - job_manager.logging_context(), - logging.ERROR, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "populated_count": populated_count, - "skipped_count": skipped_count, - "failed_count": failed_count, - } - ) diff --git a/src/mavedb/worker/jobs/external_services/variant_translation.py b/src/mavedb/worker/jobs/external_services/variant_translation.py deleted file mode 100644 index ddec4b731..000000000 --- a/src/mavedb/worker/jobs/external_services/variant_translation.py +++ /dev/null @@ -1,371 +0,0 @@ -"""ClinGen allele variant translation jobs for mapping PA<->CA allele relationships. - -This module populates the variant_translations table with relationships between -protein allele (PA) and nucleotide allele (CA) ClinGen IDs. For CA alleles, it -looks up MANE canonical PA IDs and their matching registered transcript CA IDs. -For PA alleles, it looks up matching registered transcript CA IDs directly. -""" - -import logging - -import requests -from sqlalchemy import select - -from mavedb.lib.annotation_status_manager import AnnotationStatusManager -from mavedb.lib.clingen.allele_registry import ( - expand_allele_ids, - get_canonical_pa_ids, - get_matching_registered_ca_ids, -) -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.lib.variant_translations import upsert_variant_translations -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant -from mavedb.worker.jobs.utils.setup import validate_job_params -from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management -from mavedb.worker.lib.managers.job_manager import JobManager - -logger = logging.getLogger(__name__) - - -@with_pipeline_management -async def populate_variant_translations_for_score_set( - ctx: dict, job_id: int, job_manager: JobManager -) -> JobExecutionOutcome: - """Populate variant translations (PA<->CA relationships) for a score set. - - Queries the ClinGen Allele Registry to discover relationships between protein - allele (PA) and nucleotide allele (CA) ClinGen IDs, then stores them in the - variant_translations table. Each unique allele ID is processed once even if - shared across multiple mapped variants. - - Required job_params in the JobRun: - - score_set_id (int): ID of the ScoreSet to process - - correlation_id (str): Correlation ID for tracking - """ - job = job_manager.get_job() - - _job_required_params = ["score_set_id", "correlation_id"] - validate_job_params(_job_required_params, job) - - score_set = job_manager.db.scalars(select(ScoreSet).where(ScoreSet.id == job.job_params["score_set_id"])).one() # type: ignore - correlation_id = job.job_params["correlation_id"] # type: ignore - - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "populate_variant_translations_for_score_set", - "resource": score_set.urn, - "correlation_id": correlation_id, - } - ) - job_manager.update_progress(0, 100, "Starting variant translation population.") - logger.info(msg="Started variant translation population.", extra=job_manager.logging_context()) - - # Fetch all current mapped variants with their ClinGen allele IDs - variant_rows = job_manager.db.execute( - select(Variant.id, MappedVariant.clingen_allele_id) - .join(MappedVariant, MappedVariant.variant_id == Variant.id) - .join(ScoreSet, Variant.score_set_id == ScoreSet.id) - .where(ScoreSet.id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - - if not variant_rows: - logger.warning( - msg="No current mapped variants found for this score set.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - # Deduplicate: multiple mapped variants can share the same allele ID, but we only - # need to query the ClinGen API once per unique ID. Track which variants map to each - # allele so we can record annotations for all of them after a single lookup. - allele_to_variants: dict[str, list[int]] = {} - for variant_id, clingen_allele_id in variant_rows: - if not clingen_allele_id: - continue - - for individual_id in expand_allele_ids([clingen_allele_id]): - allele_to_variants.setdefault(individual_id, []).append(variant_id) - - unique_allele_ids = list(allele_to_variants.keys()) - total_alleles = len(unique_allele_ids) - job_manager.save_to_context({"total_variants": len(variant_rows), "unique_allele_ids": total_alleles}) - - if not unique_allele_ids: - logger.warning( - msg="No ClinGen allele IDs found on mapped variants.", - extra=job_manager.logging_context(), - ) - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"translations_created": 0, "alleles_skipped": 0, "alleles_failed": 0} - ) - - job_manager.update_progress(5, 100, f"Processing {total_alleles} unique allele IDs for variant translations.") - logger.info( - "Processing %s unique allele IDs for variant translations.", - total_alleles, - extra=job_manager.logging_context(), - ) - - total_created = 0 - total_skipped = 0 - total_failed = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - - for index, allele_id in enumerate(unique_allele_ids): - if total_alleles > 0 and index % max(total_alleles // 20, 1) == 0: - progress = 5 + int((index / total_alleles) * 90) - job_manager.update_progress(progress, 100, f"Processing allele {index + 1}/{total_alleles}.") - logger.info( - "Processing allele %s/%s: %s", - index + 1, - total_alleles, - allele_id, - extra=job_manager.logging_context(), - ) - - job_manager.save_to_context( - { - "current_allele_id": allele_id, - "progress_index": index, - } - ) - - variant_ids = allele_to_variants[allele_id] - - if allele_id.startswith("CA"): - # CA (nucleotide) alleles: look up the MANE canonical protein alleles (PAs) for - # this CA, then for each PA discover all registered transcript-level CAs. This - # CA -> PA -> CA expansion builds the full translation graph so we can link - # nucleotide variants to their protein equivalents and vice versa. - try: - canonical_pa_ids = await get_canonical_pa_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for canonical PA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error looking up PA IDs for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not canonical_pa_ids: - # Noncoding variants won't have protein alleles — this is expected and not an error. - logger.debug( - "No canonical PA IDs found for %s (may be noncoding).", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No canonical PA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - created = 0 - failed = 0 - translation_pairs: set[tuple[str, str]] = set() - for pa_id in canonical_pa_ids: - # Record the direct PA <-> original CA relationship. - translation_pairs.add((pa_id, allele_id)) - - # Then expand: find all other CAs registered under this PA so we capture - # alternate transcript-level representations of the same protein change. - try: - ca_ids = await get_matching_registered_ca_ids(pa_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - pa_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - failed += 1 - continue - - for ca_id in ca_ids: - translation_pairs.add((pa_id, ca_id)) - - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED if failed > 0 else AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - "pa_lookups_failed": failed, - "pa_lookups_total": len(canonical_pa_ids), - }, - }, - current=True, - ) - - total_created += created - total_failed += failed - - elif allele_id.startswith("PA"): - # PA (protein) alleles: directly look up all registered transcript-level CAs. - # This is simpler than the CA path since we already have the protein allele. - try: - ca_ids = await get_matching_registered_ca_ids(allele_id) - except requests.exceptions.RequestException as exc: - logger.error( - "ClinGen API request failed for registered CA lookup of %s.", - allele_id, - extra=job_manager.logging_context(), - exc_info=exc, - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": f"ClinGen API error for {allele_id}: {exc}", - }, - current=True, - ) - total_failed += len(variant_ids) - continue - - if not ca_ids: - logger.warning( - "No matching registered transcript CA IDs for PA allele %s. This is unexpected.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, - annotation_data={ - "error_message": f"No registered transcript CA IDs for {allele_id}.", - }, - current=True, - ) - total_skipped += len(variant_ids) - continue - - translation_pairs = set([(allele_id, ca_id) for ca_id in ca_ids]) - created, existing = upsert_variant_translations(job_manager.db, list(translation_pairs)) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": { - "allele_id": allele_id, - "translation_pairs": [[pa, ca] for pa, ca in translation_pairs], - "translations_new": created, - "translations_existing": existing, - }, - }, - current=True, - ) - - total_created += created - - else: - logger.warning( - "Unrecognized ClinGen allele ID format: %s. Skipping.", - allele_id, - extra=job_manager.logging_context(), - ) - for vid in variant_ids: - annotation_manager.add_annotation( - variant_id=vid, - annotation_type=AnnotationType.VARIANT_TRANSLATION, - version=None, - status=AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, - annotation_data={ - "error_message": f"Unrecognized allele ID format: {allele_id}", - }, - current=True, - ) - total_skipped += len(variant_ids) - - annotation_manager.flush() - job_manager.db.flush() - - job_manager.save_to_context( - { - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) - logger.info( - "Completed variant translation population: %s created, %s skipped, %s failed.", - total_created, - total_skipped, - total_failed, - extra=job_manager.logging_context(), - ) - - if total_failed > 0 and total_created == 0: - error_message = f"All {total_failed} variant translation lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - logger.error(error_message, extra=job_manager.logging_context()) - job_manager.db.flush() - return JobExecutionOutcome.failed( - reason=error_message, - data={ - "translations_created": 0, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - }, - failure_category=FailureCategory.DEPENDENCY_FAILURE, - ) - - job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={ - "translations_created": total_created, - "alleles_skipped": total_skipped, - "alleles_failed": total_failed, - } - ) diff --git a/src/mavedb/worker/jobs/registry.py b/src/mavedb/worker/jobs/registry.py index c445098ef..8b2b712f8 100644 --- a/src/mavedb/worker/jobs/registry.py +++ b/src/mavedb/worker/jobs/registry.py @@ -18,8 +18,6 @@ from mavedb.worker.jobs.external_services import ( link_gnomad_variants, poll_uniprot_mapping_jobs_for_score_set, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, refresh_clinvar_controls, submit_score_set_mappings_to_car, @@ -49,8 +47,6 @@ submit_uniprot_mapping_jobs_for_score_set, poll_uniprot_mapping_jobs_for_score_set, link_gnomad_variants, - populate_hgvs_for_score_set, - populate_variant_translations_for_score_set, populate_vep_for_score_set, # Data management jobs refresh_materialized_views, @@ -160,20 +156,6 @@ "key": "link_gnomad_variants", "type": JobType.MAPPED_VARIANT_ANNOTATION, }, - populate_hgvs_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_hgvs_for_score_set", - "key": "populate_hgvs_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, - populate_variant_translations_for_score_set: { - "dependencies": [], - "params": {"score_set_id": None, "correlation_id": None}, - "function": "populate_variant_translations_for_score_set", - "key": "populate_variant_translations_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - }, populate_vep_for_score_set: { "dependencies": [], "params": {"score_set_id": None, "correlation_id": None}, diff --git a/tests/worker/jobs/conftest.py b/tests/worker/jobs/conftest.py index 6b7309fd2..873d09ff0 100644 --- a/tests/worker/jobs/conftest.py +++ b/tests/worker/jobs/conftest.py @@ -1011,198 +1011,6 @@ def with_cleanup_job(session, sample_cleanup_job_run): session.commit() -## HGVS Population Job Fixtures ## - - -@pytest.fixture -def populate_hgvs_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_hgvs_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_hgvs_pipeline(): - """Create a pipeline instance for populate_hgvs_for_score_set job.""" - - return Pipeline( - urn="test:populate_hgvs_pipeline", - name="Populate HGVS Pipeline", - ) - - -@pytest.fixture -def sample_populate_hgvs_run(populate_hgvs_sample_params): - """Create a JobRun instance for populate_hgvs_for_score_set job.""" - - return JobRun( - urn="test:populate_hgvs_for_score_set", - job_type="populate_hgvs_for_score_set", - job_function="populate_hgvs_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_hgvs_sample_params, - ) - - -@pytest.fixture -def with_populate_hgvs_job(session, sample_populate_hgvs_run): - """Add a populate_hgvs_for_score_set job run to the session.""" - - session.add(sample_populate_hgvs_run) - session.commit() - - -@pytest.fixture -def with_populate_hgvs_pipeline(session, sample_populate_hgvs_pipeline): - """Add a populate_hgvs pipeline to the session.""" - - session.add(sample_populate_hgvs_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_hgvs_run_pipeline( - session, - with_populate_hgvs_job, - with_populate_hgvs_pipeline, - sample_populate_hgvs_run, - sample_populate_hgvs_pipeline, -): - """Provide a context with a populate_hgvs job run and pipeline.""" - - sample_populate_hgvs_run.pipeline_id = sample_populate_hgvs_pipeline.id - session.commit() - return sample_populate_hgvs_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_hgvs( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_hgvs_run -): - """Setup variants and mapped variants in the database for HGVS population testing.""" - score_set = session.get(ScoreSet, sample_populate_hgvs_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-hgvs", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - -# --- Variant Translation Fixtures --- - - -@pytest.fixture -def populate_variant_translations_sample_params(with_populated_domain_data, sample_score_set): - """Provide sample parameters for populate_variant_translations_for_score_set job.""" - - return { - "correlation_id": "sample-correlation-id", - "score_set_id": sample_score_set.id, - } - - -@pytest.fixture -def sample_populate_variant_translations_pipeline(): - """Create a pipeline instance for populate_variant_translations_for_score_set job.""" - - return Pipeline( - urn="test:populate_variant_translations_pipeline", - name="Populate Variant Translations Pipeline", - ) - - -@pytest.fixture -def sample_populate_variant_translations_run(populate_variant_translations_sample_params): - """Create a JobRun instance for populate_variant_translations_for_score_set job.""" - - return JobRun( - urn="test:populate_variant_translations_for_score_set", - job_type="populate_variant_translations_for_score_set", - job_function="populate_variant_translations_for_score_set", - max_retries=3, - retry_count=0, - job_params=populate_variant_translations_sample_params, - ) - - -@pytest.fixture -def with_populate_variant_translations_job(session, sample_populate_variant_translations_run): - """Add a populate_variant_translations_for_score_set job run to the session.""" - - session.add(sample_populate_variant_translations_run) - session.commit() - - -@pytest.fixture -def with_populate_variant_translations_pipeline(session, sample_populate_variant_translations_pipeline): - """Add a populate_variant_translations pipeline to the session.""" - - session.add(sample_populate_variant_translations_pipeline) - session.commit() - - -@pytest.fixture -def sample_populate_variant_translations_run_pipeline( - session, - with_populate_variant_translations_job, - with_populate_variant_translations_pipeline, - sample_populate_variant_translations_run, - sample_populate_variant_translations_pipeline, -): - """Provide a context with a populate_variant_translations job run and pipeline.""" - - sample_populate_variant_translations_run.pipeline_id = sample_populate_variant_translations_pipeline.id - session.commit() - return sample_populate_variant_translations_run - - -@pytest.fixture -def setup_sample_variants_with_caid_for_translation( - session, with_populated_domain_data, mock_worker_ctx, sample_populate_variant_translations_run -): - """Setup variants and mapped variants in the database for variant translation testing.""" - score_set = session.get(ScoreSet, sample_populate_variant_translations_run.job_params["score_set_id"]) - - variant = Variant( - urn="urn:variant:test-variant-with-caid-translation", - score_set_id=score_set.id, - hgvs_nt="NM_000000.1:c.1A>G", - hgvs_pro="NP_000000.1:p.Met1Val", - data={"hgvs_c": "NM_000000.1:c.1A>G", "hgvs_p": "NP_000000.1:p.Met1Val"}, - ) - session.add(variant) - session.commit() - mapped_variant = MappedVariant( - variant_id=variant.id, - clingen_allele_id=VALID_CAID, - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant) - session.commit() - return variant, mapped_variant - - ## ClinGen Cache Warming Job Fixtures ## diff --git a/tests/worker/jobs/external_services/network/test_hgvs.py b/tests/worker/jobs/external_services/network/test_hgvs.py deleted file mode 100644 index 56f100e0b..000000000 --- a/tests/worker/jobs/external_services/network/test_hgvs.py +++ /dev/null @@ -1,54 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateHgvsForScoreSet: - """End-to-end test for HGVS population against the real ClinGen API.""" - - async def test_populate_hgvs_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Enqueue the HGVS population job, run the worker, and verify HGVS fields are populated.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - session.refresh(mapped_variant) - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.MAPPED_HGVS, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/network/test_variant_translations.py b/tests/worker/jobs/external_services/network/test_variant_translations.py deleted file mode 100644 index b45087dd9..000000000 --- a/tests/worker/jobs/external_services/network/test_variant_translations.py +++ /dev/null @@ -1,56 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from sqlalchemy import select - -from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.network -@pytest.mark.slow -class TestE2EPopulateVariantTranslationsForScoreSet: - """End-to-end test for variant translation population against the real ClinGen API.""" - - async def test_populate_variant_translations_e2e( - self, - session, - arq_redis, - arq_worker, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Enqueue the variant translation job, run the worker, and verify translations are created.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == mapped_variant.variant_id, - VariantAnnotationStatus.annotation_type == AnnotationType.VARIANT_TRANSLATION, - VariantAnnotationStatus.current.is_(True), - ) - ).one_or_none() - assert annotation is not None - assert annotation.status in (AnnotationStatus.SUCCESS, AnnotationStatus.SKIPPED) diff --git a/tests/worker/jobs/external_services/test_hgvs.py b/tests/worker/jobs/external_services/test_hgvs.py deleted file mode 100644 index 946724cc5..000000000 --- a/tests/worker/jobs/external_services/test_hgvs.py +++ /dev/null @@ -1,544 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.worker.jobs.external_services.hgvs import populate_hgvs_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - -SAMPLE_CA_ALLELE_DATA = { - "genomicAlleles": [ - { - "referenceGenome": "GRCh38", - "hgvs": ["NC_000001.11:g.12345A>G"], - } - ], - "transcriptAlleles": [ - { - "hgvs": ["NM_000000.1:c.1A>G"], - "proteinEffect": {"hgvs": "NP_000000.1:p.Met1Val"}, - "MANE": { - "nucleotide": {"RefSeq": {"hgvs": "NM_000000.1:c.1A>G"}}, - "protein": {"RefSeq": {"hgvs": "NP_000000.1:p.Met1Val"}}, - }, - } - ], -} - -SAMPLE_PA_ALLELE_DATA = { - "aminoAcidAlleles": [ - { - "hgvs": ["NP_000000.1:p.Met1Val"], - } - ], -} - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateHgvsForScoreSetUnit: - """Unit tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test populating HGVS when no mapped variants exist.""" - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - async def test_variant_without_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant without a CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_variant_with_multi_caid_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a variant with a multi-variant CAID gets a skipped annotation.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = "CA123,CA456" - session.commit() - - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_successful_ca_allele_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test successful HGVS population for a CA allele.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["populated_count"] == 1 - - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - session.refresh(mapped_variant) - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - async def test_clingen_api_error_recorded_as_failed( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that ClinGen API errors are recorded as failed annotations.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - - async def test_total_api_failure_sends_slack_alert( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a Slack alert is sent when all variants fail HGVS population.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=requests.exceptions.ConnectionError("Connection refused"), - ), - patch("mavedb.worker.jobs.external_services.hgvs.log_and_send_slack_message") as mock_slack, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["failed_count"] == 1 - assert result.data["populated_count"] == 0 - mock_slack.assert_called_once() - - async def test_clingen_allele_not_found_skipped( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that a 404 from ClinGen results in a skipped annotation.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=None, - ), - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["skipped_count"] == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_hgvs_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_hgvs_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetIntegration: - """Integration tests for the populate_hgvs_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end successful HGVS population.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated with HGVS - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_successful_hgvs_population_pipeline( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test end-to-end HGVS population in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run_pipeline.id) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that variants without CAIDs get a skipped annotation status.""" - _, mapped_variant = setup_sample_variants_with_caid_for_hgvs - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_hgvs_for_score_set(mock_worker_ctx, sample_populate_hgvs_run.id) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_hgvs_job, - mock_worker_ctx, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_hgvs_for_score_set( - mock_worker_ctx, - sample_populate_hgvs_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateHgvsForScoreSetArqContext: - """Tests for populate_hgvs_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job completed - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.SUCCEEDED - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_run_pipeline, - sample_populate_hgvs_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - return_value=SAMPLE_CA_ALLELE_DATA, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - # Verify mapped variant was updated - mapped_variant = session.query(MappedVariant).first() - assert mapped_variant.hgvs_g == "NC_000001.11:g.12345A>G" - - # Verify annotation status - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "mapped_hgvs" - - # Verify job and pipeline status - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_hgvs_job, - sample_populate_hgvs_run, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run) - assert sample_populate_hgvs_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_hgvs_pipeline, - sample_populate_hgvs_run_pipeline, - setup_sample_variants_with_caid_for_hgvs, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.hgvs.get_clingen_allele_data", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job("populate_hgvs_for_score_set", sample_populate_hgvs_run_pipeline.id) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - # Verify job errored - session.refresh(sample_populate_hgvs_run_pipeline) - assert sample_populate_hgvs_run_pipeline.status == JobStatus.ERRORED - - # Verify pipeline failed - session.refresh(sample_populate_hgvs_pipeline) - assert sample_populate_hgvs_pipeline.status == PipelineStatus.FAILED diff --git a/tests/worker/jobs/external_services/test_variant_translation.py b/tests/worker/jobs/external_services/test_variant_translation.py deleted file mode 100644 index 3e1f364ea..000000000 --- a/tests/worker/jobs/external_services/test_variant_translation.py +++ /dev/null @@ -1,770 +0,0 @@ -# ruff: noqa: E402 - -import pytest - -pytest.importorskip("arq") - -from unittest.mock import patch - -from sqlalchemy import select - -from mavedb.lib.types.workflow import JobExecutionOutcome -from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus, PipelineStatus -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus -from mavedb.models.variant_translation import VariantTranslation -from mavedb.worker.jobs.external_services.variant_translation import populate_variant_translations_for_score_set -from mavedb.worker.lib.managers.job_manager import JobManager - -pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") - - -# --- Unit Tests --- - - -@pytest.mark.asyncio -@pytest.mark.unit -class TestPopulateVariantTranslationsUnit: - """Unit tests for the populate_variant_translations_for_score_set job.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test that the job succeeds with zero translations when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_variant_without_caid_no_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a variant without a CAID results in no translations.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - async def test_ca_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele creates translations via PA lookup.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00001"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA11111", "CA22222"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # 1 for PA00001->CA9765210 (the original CA), 2 for PA00001->CA11111 and PA00001->CA22222 - assert result.data["translations_created"] == 3 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 3 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation is not None - - async def test_pa_allele_creates_translations( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele creates translations via CA lookup.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA99999" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA33333", "CA44444"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 2 - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 - aa_ids = {t.aa_clingen_id for t in translations} - assert aa_ids == {"PA99999"} - - async def test_multi_variant_caid_expanded( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that comma-separated CAIDs are expanded and each processed independently.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "CA55555,CA66666" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00002"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - # PA00002->CA55555 and PA00002->CA66666 - assert result.data["translations_created"] == 2 - - async def test_ca_allele_no_pa_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a CA allele with no canonical PA IDs results in a skip.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "skipped" - - async def test_pa_allele_no_ca_ids_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a PA allele with no registered CA IDs results in a skip.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "PA88888" - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - assert result.data["translations_created"] == 0 - - async def test_ca_allele_api_failure_records_failed_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a ClinGen API failure for CA allele records a failed annotation.""" - import requests - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - - annotation = session.scalars(select(VariantAnnotationStatus)).one() - assert annotation.status == "failed" - - async def test_unrecognized_allele_format_skipped( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that an unrecognized allele ID format is skipped.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["alleles_skipped"] == 1 - - async def test_duplicate_translations_not_created( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that duplicate translations are not created on re-run.""" - # Pre-populate a translation - session.add(VariantTranslation(aa_clingen_id="PA00003", nt_clingen_id="CA9765210")) - session.commit() - - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00003"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - translations = session.scalars( - select(VariantTranslation).where(VariantTranslation.aa_clingen_id == "PA00003") - ).all() - assert len(translations) == 1 - - async def test_propagates_exceptions( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are propagated.""" - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ): - with pytest.raises(Exception) as exc_info: - await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert str(exc_info.value) == "Test exception" - - async def test_multiple_alleles_sharing_pa_no_duplicate_error( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that two CA alleles mapping to the same PA don't cause a UniqueViolation. - - This is a regression test for a bug where the SELECT-then-INSERT upsert pattern - failed to detect in-session duplicates: both alleles' iterations called - upsert_variant_translations with overlapping (PA, CA) pairs, the SELECT found - no committed row, both staged db.add() for the same pair, and the subsequent - update_progress commit raised a UniqueViolation. - """ - # Add a second variant with a different CA allele under the same score set. - score_set_id = sample_populate_variant_translations_run.job_params["score_set_id"] - - variant2 = Variant( - urn="urn:variant:test-second-ca-allele", - score_set_id=score_set_id, - hgvs_nt="NM_000000.1:c.2T>G", - hgvs_pro="NP_000000.1:p.Val2Gly", - data={}, - ) - session.add(variant2) - session.commit() - mapped_variant2 = MappedVariant( - variant_id=variant2.id, - clingen_allele_id="CA_SECOND", - current=True, - mapped_date="2024-01-01T00:00:00Z", - mapping_api_version="1.0.0", - ) - session.add(mapped_variant2) - session.commit() - - # Both CA alleles resolve to the same PA. The PA then returns the same set of - # registered CAs for both iterations, producing fully overlapping translation pairs. - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA_SHARED"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA9765210", "CA_SECOND"], - ), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - pairs = {(t.aa_clingen_id, t.nt_clingen_id) for t in translations} - # PA_SHARED paired with each CA: CA9765210 (original from allele 1), - # CA_SECOND (original from allele 2), plus both as registered CAs. - assert ("PA_SHARED", "CA9765210") in pairs - assert ("PA_SHARED", "CA_SECOND") in pairs - - async def test_total_api_failure_returns_failed( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job returns FAILED when all variant translation lookups fail.""" - import requests - - with patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=requests.exceptions.ConnectionError("Connection failed"), - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - 1, - JobManager(session, mock_worker_ctx["redis"], sample_populate_variant_translations_run.id), - ) - - assert result.status == JobStatus.FAILED - assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - assert result.data["alleles_failed"] == 1 - assert result.data["translations_created"] == 0 - - -# --- Integration Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsIntegration: - """Integration tests that exercise the full decorator stack.""" - - async def test_no_mapped_variants( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - ): - """Test end-to-end when no mapped variants exist.""" - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_successful_job_updates_status( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a successful job run updates the job status to SUCCEEDED.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00004"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA77777"], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00004->CA9765210 and PA00004->CA77777 - - async def test_job_with_pipeline_updates_pipeline_status( - self, - session, - with_populated_domain_data, - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that a job in a pipeline updates the pipeline status on success.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00005"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run_pipeline.id, - ) - - session.refresh(sample_populate_variant_translations_run_pipeline) - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_variant_without_caid_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that variants without CAIDs produce no annotations (filtered before processing).""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = None - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - assert result.data["translations_created"] == 0 - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_unrecognized_allele_creates_skipped_annotation( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unrecognized allele formats create skipped annotations through the full stack.""" - _, mapped_variant = setup_sample_variants_with_caid_for_translation - mapped_variant.clingen_allele_id = "XX12345" - session.commit() - - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, sample_populate_variant_translations_run.id - ) - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.SUCCEEDED - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - async def test_exceptions_handled_by_decorators( - self, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - mock_worker_ctx, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that unexpected exceptions are handled by decorators.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - result = await populate_variant_translations_for_score_set( - mock_worker_ctx, - sample_populate_variant_translations_run.id, - ) - - mock_send_slack_job_error.assert_called_once() - assert isinstance(result, JobExecutionOutcome) - assert result.status == JobStatus.ERRORED - assert isinstance(result.exception, Exception) - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - -# --- ARQ Context Tests --- - - -@pytest.mark.asyncio -@pytest.mark.integration -class TestPopulateVariantTranslationsArqContext: - """Tests for populate_variant_translations_for_score_set job using the ARQ context fixture.""" - - async def test_with_arq_context_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00006"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=["CA88888"], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.SUCCEEDED - - translations = session.scalars(select(VariantTranslation)).all() - assert len(translations) == 2 # PA00006->CA9765210 and PA00006->CA88888 - - async def test_with_arq_context_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_run_pipeline, - sample_populate_variant_translations_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that the job works with the ARQ context fixture in a pipeline.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - return_value=["PA00007"], - ), - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_matching_registered_ca_ids", - return_value=[], - ), - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "variant_translation" - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.SUCCEEDED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.SUCCEEDED - - async def test_with_arq_context_exception_handling_independent( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - with_populate_variant_translations_job, - sample_populate_variant_translations_run, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions are handled with the ARQ context fixture.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run) - assert sample_populate_variant_translations_run.status == JobStatus.ERRORED - - async def test_with_arq_context_exception_handling_pipeline( - self, - arq_redis, - arq_worker, - session, - with_populated_domain_data, - sample_populate_variant_translations_pipeline, - sample_populate_variant_translations_run_pipeline, - setup_sample_variants_with_caid_for_translation, - ): - """Test that exceptions in pipeline context are handled.""" - with ( - patch( - "mavedb.worker.jobs.external_services.variant_translation.get_canonical_pa_ids", - side_effect=Exception("Test exception"), - ), - patch("mavedb.worker.lib.decorators.job_management.send_slack_job_error") as mock_send_slack_job_error, - ): - await arq_redis.enqueue_job( - "populate_variant_translations_for_score_set", - sample_populate_variant_translations_run_pipeline.id, - ) - await arq_worker.async_run() - await arq_worker.run_check() - - mock_send_slack_job_error.assert_called_once() - - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 - - session.refresh(sample_populate_variant_translations_run_pipeline) - assert sample_populate_variant_translations_run_pipeline.status == JobStatus.ERRORED - - session.refresh(sample_populate_variant_translations_pipeline) - assert sample_populate_variant_translations_pipeline.status == PipelineStatus.FAILED From c936dd4735556a74d0174eb5450a6b5eb772ca97 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 23 Jun 2026 09:27:43 -0700 Subject: [PATCH 29/93] feat(alleles): cross-layer allele equivalence query with as_of support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_allele_translations(db, allele_id, *, as_of=None) returns an allele's full cross-layer equivalence set (genomic/coding/protein) by traversing the MappingRecordAllele link graph: allele -> its live links -> mapping record(s) -> all co-linked alleles. The relation is co-membership in a MappingRecord's allele set, not a shared identifier — ClinGen's CAID spans only the nucleotide layers (the protein allele carries a distinct PA), so the link graph is the only thing tying all layers together. This replaces what the retired variant_translations PA<->CA table provided. Forward-compatible with temporal reads: the same half-open valid-time predicate is applied at both the anchor and fan-out hops, so passing as_of reconstructs the equivalence set as of any instant. Defaults to the currently-live set. --- src/mavedb/lib/alleles.py | 54 +++++++++++++++++++ tests/worker/jobs/test_allele_translations.py | 41 ++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/mavedb/lib/alleles.py create mode 100644 tests/worker/jobs/test_allele_translations.py diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py new file mode 100644 index 000000000..ddf51565b --- /dev/null +++ b/src/mavedb/lib/alleles.py @@ -0,0 +1,54 @@ +"""Allele-graph queries over the deduplicated allele model. + +These traverse the ``MappingRecordAllele`` link graph rather than any external identifier, because an +allele's cross-layer equivalence (its genomic / coding / protein representations) is established by the +mapping + reverse-translation process and materialized as co-membership in a ``MappingRecord``'s allele +set. No single identifier spans the layers: ClinGen's canonical allele id (CAID) covers only the +nucleotide layers, the protein allele carries a distinct PA, so the link graph is the only thing that +ties all three together. +""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: + """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a + ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, + and protein layers. + + The relation is co-membership in a record's allele set, not a shared identifier — see the module + docstring. Because alleles are deduplicated by ``vrs_digest`` and shared across variants/score sets, + the anchor may belong to several ``MappingRecord``s; the result is the union of their allele sets + (normally the same biological equivalence class). Scope by record or score set upstream if a single + context is required. + + Temporal: by default returns the currently-live set (``valid_to IS NULL``). Pass ``as_of`` to + reconstruct the set as it stood at a past instant — both the anchor hop and the fan-out hop apply + the same half-open ``[valid_from, valid_to)`` predicate, so the whole set is evaluated at one + instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, + so filtering the links alone is sufficient. + """ + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id).where(MappingRecordAllele.allele_id == allele_id).where(link_live) + ).all() + if not record_ids: + return [] + + return list( + db.scalars( + select(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(link_live) + .distinct() + ).all() + ) diff --git a/tests/worker/jobs/test_allele_translations.py b/tests/worker/jobs/test_allele_translations.py new file mode 100644 index 000000000..0e1c1b12c --- /dev/null +++ b/tests/worker/jobs/test_allele_translations.py @@ -0,0 +1,41 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("arq") + +from datetime import datetime, timezone + +from mavedb.lib.alleles import get_allele_translations + + +@pytest.mark.unit +class TestGetAlleleTranslations: + """The cross-layer equivalence query traverses the MappingRecordAllele link graph. + + Uses the RT-derived fixture, which links an authoritative allele and a derived (cross-layer) + allele to one MappingRecord — exactly the co-membership the query resolves. + """ + + def test_returns_full_equivalence_set_from_any_member(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + from_authoritative = {a.id for a in get_allele_translations(session, authoritative_allele.id)} + from_rt = {a.id for a in get_allele_translations(session, rt_allele.id)} + + # Reachable from either member, and includes the anchor itself — the full equivalence set. + assert from_authoritative == {authoritative_allele.id, rt_allele.id} + assert from_rt == {authoritative_allele.id, rt_allele.id} + + def test_as_of_before_links_existed_is_empty(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, _rt = setup_rt_derived_allele_with_caid + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + assert get_allele_translations(session, authoritative_allele.id, as_of=past) == [] + + def test_as_of_after_links_existed_returns_set(self, session, setup_rt_derived_allele_with_caid): + _variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + ids = {a.id for a in get_allele_translations(session, authoritative_allele.id, as_of=future)} + assert ids == {authoritative_allele.id, rt_allele.id} From d397398ee8e4bb5697755df312cdfc941770b83d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:29:21 -0700 Subject: [PATCH 30/93] feat(annotation): add append-only AnnotationEvent log and status vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces the AnnotationEvent log: a single append-only event table whose subjects are Variant and Allele, selected by annotation_type via a polymorphic CHECK. "Current" is derived (DISTINCT ON … id DESC), never stored. Adds the shared Disposition (present/absent/not_applicable/failed) and EventReason vocabulary, the v_current_annotation_events view, and the supporting migrations. --- .../a7e1c4f9b3d2_add_annotation_event.py | 79 ++++++++++ ...e4_add_v_current_annotation_events_view.py | 30 ++++ src/mavedb/models/__init__.py | 2 + src/mavedb/models/annotation_event.py | 138 ++++++++++++++++ src/mavedb/models/annotation_event_view.py | 96 ++++++++++++ src/mavedb/models/enums/disposition.py | 19 +++ src/mavedb/models/enums/event_reason.py | 42 +++++ tests/models/conftest.py | 80 ++++++++++ tests/models/test_annotation_event_model.py | 103 ++++++++++++ tests/models/test_annotation_event_view.py | 147 ++++++++++++++++++ 10 files changed, 736 insertions(+) create mode 100644 alembic/versions/a7e1c4f9b3d2_add_annotation_event.py create mode 100644 alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py create mode 100644 src/mavedb/models/annotation_event.py create mode 100644 src/mavedb/models/annotation_event_view.py create mode 100644 src/mavedb/models/enums/disposition.py create mode 100644 src/mavedb/models/enums/event_reason.py create mode 100644 tests/models/conftest.py create mode 100644 tests/models/test_annotation_event_model.py create mode 100644 tests/models/test_annotation_event_view.py diff --git a/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py new file mode 100644 index 000000000..cf8d77e35 --- /dev/null +++ b/alembic/versions/a7e1c4f9b3d2_add_annotation_event.py @@ -0,0 +1,79 @@ +"""add annotation_event log + +Revision ID: a7e1c4f9b3d2 +Revises: d4e5f6a7b8c9 +Create Date: 2026-06-25 16:00:00.000000 + +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a7e1c4f9b3d2" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + +_VARIANT_SUBJECT_TYPES = "'vrs_mapping', 'cross_level_translation', 'variant_translation', 'ldh_submission'" +_ALLELE_SUBJECT_TYPES = ( + "'clingen_allele_id', 'gnomad_allele_frequency', 'vep_functional_consequence', 'clinvar_control', 'mapped_hgvs'" +) + + +def upgrade(): + op.create_table( + "annotation_event", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("annotation_type", sa.String(length=50), nullable=False), + sa.Column("variant_id", sa.Integer(), nullable=True), + sa.Column("allele_id", sa.Integer(), nullable=True), + sa.Column("disposition", sa.String(length=50), nullable=False), + sa.Column("reason", sa.String(length=50), nullable=False), + sa.Column("source_version", sa.String(length=50), nullable=True), + sa.Column("metadata", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("job_run_id", sa.Integer(), nullable=True), + sa.Column("score_set_id", sa.Integer(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.CheckConstraint( + f"(annotation_type IN ({_VARIANT_SUBJECT_TYPES}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_ALLELE_SUBJECT_TYPES}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + sa.ForeignKeyConstraint(["variant_id"], ["variants.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["allele_id"], ["alleles.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["job_run_id"], ["job_runs.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["score_set_id"], ["scoresets.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_annotation_event_allele_type_id", + "annotation_event", + ["allele_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_variant_type_id", + "annotation_event", + ["variant_id", "annotation_type", sa.text("id DESC")], + unique=False, + ) + op.create_index( + "ix_annotation_event_allele_type_version", + "annotation_event", + ["allele_id", "annotation_type", "source_version"], + unique=False, + ) + op.create_index("ix_annotation_event_job_run_id", "annotation_event", ["job_run_id"], unique=False) + + +def downgrade(): + op.drop_index("ix_annotation_event_job_run_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_version", table_name="annotation_event") + op.drop_index("ix_annotation_event_variant_type_id", table_name="annotation_event") + op.drop_index("ix_annotation_event_allele_type_id", table_name="annotation_event") + op.drop_table("annotation_event") diff --git a/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py new file mode 100644 index 000000000..73843866c --- /dev/null +++ b/alembic/versions/b8f2c5a1d3e4_add_v_current_annotation_events_view.py @@ -0,0 +1,30 @@ +"""add v_current_annotation_events view + +Revision ID: b8f2c5a1d3e4 +Revises: a7e1c4f9b3d2 +Create Date: 2026-06-26 + +Creates the v_current_annotation_events view: the latest AnnotationEvent per (subject, annotation_type), +with ClinVar partitioned additionally by source_version (multi-live, one current row per release). +Replaces the per-variant v_variant_annotations as the current-state projection over the new +allele-model annotation log. Intended for operator queries, BI, and the annotation CLI scripts. +""" + +from alembic import op + +from mavedb.db.view import CreateView, DropView +from mavedb.models.annotation_event_view import definition, signature + +# revision identifiers, used by Alembic. +revision = "b8f2c5a1d3e4" +down_revision = "a7e1c4f9b3d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute(CreateView(signature, definition, materialized=False)) + + +def downgrade() -> None: + op.execute(DropView(signature, materialized=False)) diff --git a/src/mavedb/models/__init__.py b/src/mavedb/models/__init__.py index c181e4099..1cec88e97 100644 --- a/src/mavedb/models/__init__.py +++ b/src/mavedb/models/__init__.py @@ -42,6 +42,8 @@ "uniprot_offset", "user", "variant_annotation_status", + "annotation_event", + "annotation_event_view", "variant", "variant_translation", "vep_allele_consequence", diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py new file mode 100644 index 000000000..eae8877a0 --- /dev/null +++ b/src/mavedb/models/annotation_event.py @@ -0,0 +1,138 @@ +""" +SQLAlchemy model for the annotation event log. + +One append-only log spanning the whole pipeline (mapping, reverse translation, +annotation). Its subjects are exactly two persistent entities — ``Variant`` and +``Allele`` — selected by ``annotation_type`` via the polymorphic-subject CHECK. +Everything else the pipeline touches (``MappingRecord``, ``MappingRecordAllele``, +the external value tables) is a vehicle or resolution path, never a status subject. + +This is deliberately **not** a ``ValidTime`` table. A SCD-2 state table is a lossy +projection of an event log — it discards the skip/reconfirm/no-op events that are +the audit point. "Current" is derived (``DISTINCT ON (subject, annotation_type) +… id DESC``), never a stored ``current`` flag. +""" + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, Optional + +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.mutable import MutableDict +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from mavedb.db.base import Base +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason + +if TYPE_CHECKING: + from mavedb.models.allele import Allele + from mavedb.models.job_run import JobRun + from mavedb.models.score_set import ScoreSet + from mavedb.models.variant import Variant + +VARIANT_SUBJECT_TYPES = ( + AnnotationType.VRS_MAPPING.value, + AnnotationType.CROSS_LEVEL_TRANSLATION.value, + AnnotationType.VARIANT_TRANSLATION.value, + AnnotationType.LDH_SUBMISSION.value, +) +"""annotation_type values whose subject is the variant (variant_id set, allele_id null)""" + +ALLELE_SUBJECT_TYPES = ( + AnnotationType.CLINGEN_ALLELE_ID.value, + AnnotationType.GNOMAD_ALLELE_FREQUENCY.value, + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE.value, + AnnotationType.CLINVAR_CONTROL.value, + AnnotationType.MAPPED_HGVS.value, +) +"""annotation_type values whose subject is the allele (allele_id set, variant_id null)""" + + +def _sql_in_list(values: tuple[str, ...]) -> str: + return ", ".join(f"'{v}'" for v in values) + + +class AnnotationEvent(Base): + """An append-only event recording the *status* (not value) of one pipeline + observation about a ``Variant`` or an ``Allele``. + + The value (frequency, consequence, CAID, control) lives in the domain + ValidTime tables; this log records disposition + why + when. Reading the + domain tables alone cannot distinguish confirmed-absence from never-checked — + that gap is the whole reason the log exists. + + NOTE: JSONB ``event_metadata`` is tracked as a mutable object via MutableDict, + which only catches top-level mutations. Mutating a nested object + requires ``flag_modified(instance, "event_metadata")``. + """ + + __tablename__ = "annotation_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + + annotation_type: Mapped[AnnotationType] = mapped_column(String(50), nullable=False) + + # Exactly one is set, per ck_annotation_event_subject. + variant_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("variants.id", ondelete="RESTRICT"), nullable=True + ) + allele_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=True + ) + + disposition: Mapped[Disposition] = mapped_column(String(50), nullable=False) + + # Domain-specific code reusing in-code vocabularies (EventReason, plus MappingOutcome and RT + # skip_category for those two jobs); disposition is the public axis. + reason: Mapped[EventReason] = mapped_column(String(50), nullable=False) + + # gnomAD db_version / Ensembl release / ClinVar release / mapper version. + source_version: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + + # DB column is "metadata"; the attribute avoids the reserved Declarative name. + event_metadata: Mapped[Optional[Dict[str, Any]]] = mapped_column( + "metadata", MutableDict.as_mutable(JSONB), nullable=True + ) + + job_run_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("job_runs.id", ondelete="SET NULL"), nullable=True + ) + score_set_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("scoresets.id", ondelete="SET NULL"), nullable=True + ) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + # One-directional (no back-ref) — match the annotation-link convention. + variant: Mapped[Optional["Variant"]] = relationship("Variant") + allele: Mapped[Optional["Allele"]] = relationship("Allele") + job_run: Mapped[Optional["JobRun"]] = relationship("JobRun") + score_set: Mapped[Optional["ScoreSet"]] = relationship("ScoreSet") + + __table_args__ = ( + # Polymorphic subject: the type picks exactly one subject column. + CheckConstraint( + f"(annotation_type IN ({_sql_in_list(VARIANT_SUBJECT_TYPES)}) " + "AND variant_id IS NOT NULL AND allele_id IS NULL) " + f"OR (annotation_type IN ({_sql_in_list(ALLELE_SUBJECT_TYPES)}) " + "AND allele_id IS NOT NULL AND variant_id IS NULL)", + name="ck_annotation_event_subject", + ), + # latest-per-allele / latest-per-variant (the DISTINCT ON … id DESC projections) + Index("ix_annotation_event_allele_type_id", "allele_id", "annotation_type", text("id DESC")), + Index("ix_annotation_event_variant_type_id", "variant_id", "annotation_type", text("id DESC")), + # version-keyed skip + Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), + # audit by run + Index("ix_annotation_event_job_run_id", "job_run_id"), + ) + + def __repr__(self) -> str: + subject = f"variant_id={self.variant_id}" if self.variant_id is not None else f"allele_id={self.allele_id}" + return ( + f"" + ) diff --git a/src/mavedb/models/annotation_event_view.py b/src/mavedb/models/annotation_event_view.py new file mode 100644 index 000000000..5c8547807 --- /dev/null +++ b/src/mavedb/models/annotation_event_view.py @@ -0,0 +1,96 @@ +"""``v_current_annotation_events`` — the current-state projection over the AnnotationEvent log. + +The log is append-only; "current" is derived, never stored. This view exposes the latest event per +``(subject, annotation_type)`` — one row per allele/variant per annotation type — so operators, BI, +and app code can read current status with a plain ``SELECT`` instead of re-deriving the +``DISTINCT ON`` everywhere. It mirrors the existing ``v_variant_annotations`` pattern. + +ClinVar is **multi-live**: an allele accumulates one live link per archival release, so its current +status is one row *per release*. The window partition folds ``source_version`` in **only** for +``clinvar_control`` (via a CASE that is constant-NULL for every other type, collapsing those back to +one row per subject+type). + +The subject is polymorphic — exactly one of ``variant_id`` / ``allele_id`` is set per row (enforced by +the log's CHECK), and the other is constant-NULL within a partition, so partitioning by both keys each +row to its real subject. + +There is deliberately **no ``score_set_id`` axis**: an allele-subject status (CAID, gnomAD, ClinVar, +VEP) is a *shared* allele-level fact, not a property of any one score set. A consumer that wants a +score set's annotation status resolves the score set's current alleles (and variants) through the live +mapping links — e.g. ``lib.clingen.alleles.get_alleles_for_score_set`` — and then looks those subjects +up here. Filtering by the *run's* score set would wrongly drop an allele last (re-)annotated by another +score set's run. The derived "current-for-variant" walk (resolving an allele-subject fact down to a +variant at the level a type keys on) is intentionally **not** built here — that is the deferred +consumer surface; see ``docs/design/allele-annotation-status.md``. +""" + +from sqlalchemy import case, func, select + +from mavedb.db.base import Base +from mavedb.db.view import view +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType + +signature = "v_current_annotation_events" + +# ClinVar is the lone multi-live type: split "current" by release. For every other type this is NULL, +# so the partition collapses to (allele_id, variant_id, annotation_type) — one current row per subject. +_clinvar_release_key = case( + (AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL.value, AnnotationEvent.source_version), + else_=None, +) + +_ranked = select( + AnnotationEvent.id.label("id"), + AnnotationEvent.annotation_type.label("annotation_type"), + AnnotationEvent.variant_id.label("variant_id"), + AnnotationEvent.allele_id.label("allele_id"), + AnnotationEvent.disposition.label("disposition"), + AnnotationEvent.reason.label("reason"), + AnnotationEvent.source_version.label("source_version"), + AnnotationEvent.event_metadata.label("event_metadata"), + AnnotationEvent.job_run_id.label("job_run_id"), + AnnotationEvent.created_at.label("created_at"), + func.row_number() + .over( + partition_by=[ + AnnotationEvent.allele_id, + AnnotationEvent.variant_id, + AnnotationEvent.annotation_type, + _clinvar_release_key, + ], + order_by=AnnotationEvent.id.desc(), + ) + .label("row_number"), +).subquery("ranked_annotation_events") + +definition = select( + _ranked.c.id, + _ranked.c.annotation_type, + _ranked.c.variant_id, + _ranked.c.allele_id, + _ranked.c.disposition, + _ranked.c.reason, + _ranked.c.source_version, + _ranked.c.event_metadata, + _ranked.c.job_run_id, + _ranked.c.created_at, +).where(_ranked.c.row_number == 1) + + +class CurrentAnnotationEventView(Base): + __table__ = view(signature, definition, materialized=False) + # Each surviving event id is unique across the view, so it is a valid mapping key for the + # otherwise-PK-less view (standard SQLAlchemy view-mapping idiom). + __mapper_args__ = {"primary_key": [__table__.c.id]} + + id = __table__.c.id + annotation_type = __table__.c.annotation_type + variant_id = __table__.c.variant_id + allele_id = __table__.c.allele_id + disposition = __table__.c.disposition + reason = __table__.c.reason + source_version = __table__.c.source_version + event_metadata = __table__.c.event_metadata + job_run_id = __table__.c.job_run_id + created_at = __table__.c.created_at diff --git a/src/mavedb/models/enums/disposition.py b/src/mavedb/models/enums/disposition.py new file mode 100644 index 000000000..94438f292 --- /dev/null +++ b/src/mavedb/models/enums/disposition.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class Disposition(str, Enum): + """The stable, consumer-facing status axis of a variant event. + + Defined by *what the consumer may conclude* — not by the domain-specific + operation that produced it (that lives in ``reason``). + + - ``present`` — we hold the result / the step succeeded + - ``absent`` — the source or biology has nothing — an informative negative + - ``not_applicable`` — we could not ask — a pipeline/structural gap, not a statement about the source + - ``failed`` — errored, might retry; failure_category carries transient vs permanent + """ + + PRESENT = "present" + ABSENT = "absent" + NOT_APPLICABLE = "not_applicable" + FAILED = "failed" diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py new file mode 100644 index 000000000..f7fd4b56b --- /dev/null +++ b/src/mavedb/models/enums/event_reason.py @@ -0,0 +1,42 @@ +from enum import Enum + + +class EventReason(str, Enum): + """Pipeline-wide vocabulary for a variant event's ``reason`` — the single field that says + *what happened*, spanning present / absent / not_applicable / failed. + + Reasons are shared across jobs wherever they mean the same thing (the ``annotation_type`` + already says *which* job), and job-specific only where a case is genuinely unique. One job + contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses + ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the + external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + """ + + # present — we hold the result / the step succeeded + CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) + PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) + RECONFIRMED = "reconfirmed" # re-verified unchanged (gnomAD, CAR force) + SKIPPED = "skipped" # version-skip: already current at this source version (gnomAD, VEP) + SUPERSEDED = "superseded" # re-resolved within a release, newest wins (ClinVar) + SUBMITTED = "submitted" # LDH + TRANSLATED = "translated" # reverse translation + RECONFIRMATION_SKIPPED = "reconfirmation_skipped" # HGVS no longer buildable, existing CAID kept (CAR) + + # absent — the source or biology has nothing (informative negative) + NO_RECORD = "no_record" # source queried, returned nothing (gnomAD, ClinVar, VEP) + NO_CODING_TRANSCRIPT = "no_coding_transcript" # non-coding target has no protein consequence (RT) + + # not_applicable — we could not ask (structural gap) + NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) + NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) + MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) + + # failed — errored + API_ERROR = "api_error" # network/timeout/upstream error (ClinVar, VEP, LDH, CAR no-response) + SERVICE_REJECTED = "service_rejected" # external service refused the input (CAR) + MALFORMED_RESPONSE = "malformed_response" # unparseable/contractless response (CAR) + CAID_CONFLICT = "caid_conflict" # returned identifier conflicts with the stored one (CAR) + TRANSLATION_FAILED = "translation_failed" # all candidate HGVS failed translation (RT) + TRANSLATION_ERROR = "translation_error" # the translation engine errored (RT) + TRANSCRIPT_UNRESOLVED = "transcript_unresolved" # protein-coding target with no resolvable transcript (RT) diff --git a/tests/models/conftest.py b/tests/models/conftest.py new file mode 100644 index 000000000..5edfe06ee --- /dev/null +++ b/tests/models/conftest.py @@ -0,0 +1,80 @@ +import pytest + +from mavedb.models.enums import JobStatus +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.job_run import JobRun +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_EXPERIMENT, + TEST_EXPERIMENT_SET, + TEST_LICENSE, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, + TEST_USER, + VALID_EXPERIMENT_SET_URN, + VALID_EXPERIMENT_URN, + VALID_SCORE_SET_URN, +) + + +@pytest.fixture +def setup_lib_db_with_score_set(session, setup_lib_db): + """Build an experiment set, experiment, and score set on top of the base lib db (users/licenses).""" + user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + experiment_set = ExperimentSet(**TEST_EXPERIMENT_SET, urn=VALID_EXPERIMENT_SET_URN) + experiment_set.created_by = user + experiment_set.modified_by = user + session.add(experiment_set) + session.commit() + session.refresh(experiment_set) + + experiment = Experiment(**TEST_EXPERIMENT, urn=VALID_EXPERIMENT_URN, experiment_set_id=experiment_set.id) + experiment.created_by = user + experiment.modified_by = user + session.add(experiment) + session.commit() + session.refresh(experiment) + + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, urn=VALID_SCORE_SET_URN, experiment_id=experiment.id, licence_id=TEST_LICENSE["id"] + ) + score_set.created_by = user + score_set.modified_by = user + session.add(score_set) + session.commit() + session.refresh(score_set) + + return score_set + + +@pytest.fixture +def setup_lib_db_with_variant(session, setup_lib_db_with_score_set): + """Add a single variant to the score set, for variant-subject event tests.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#1", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + return variant + + +@pytest.fixture +def job_run(session): + """Create a persisted JobRun to anchor an event's provenance.""" + job = JobRun( + job_type="test_annotation_job", + job_function="test_function", + status=JobStatus.RUNNING, + ) + session.add(job) + session.commit() + session.refresh(job) + return job diff --git a/tests/models/test_annotation_event_model.py b/tests/models/test_annotation_event_model.py new file mode 100644 index 000000000..336536479 --- /dev/null +++ b/tests/models/test_annotation_event_model.py @@ -0,0 +1,103 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy.exc import IntegrityError + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +@pytest.fixture +def allele(session): + allele = Allele(vrs_digest="test-variant-event-allele-digest", level="genomic") + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _expect_rejected(session, event): + """Adding `event` must violate a CHECK constraint on flush.""" + session.add(event) + with pytest.raises(IntegrityError): + session.flush() + session.rollback() + + +class TestSubjectConstraint: + def test_variant_subject_event_inserts(self, session, setup_lib_db_with_variant, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + score_set_id=setup_lib_db_with_variant.score_set_id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.allele_id is None + + def test_allele_subject_event_inserts(self, session, allele, job_run): + event = AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason="no_record", + source_version="4.1.0", + job_run_id=job_run.id, + ) + session.add(event) + session.commit() + assert event.id is not None + assert event.variant_id is None + + def test_variant_subject_type_with_allele_id_rejected(self, session, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_allele_subject_type_with_variant_id_rejected(self, session, setup_lib_db_with_variant): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason="created", + ), + ) + + def test_neither_subject_set_rejected(self, session): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) + + def test_both_subjects_set_rejected(self, session, setup_lib_db_with_variant, allele): + _expect_rejected( + session, + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="mapped", + ), + ) diff --git a/tests/models/test_annotation_event_view.py b/tests/models/test_annotation_event_view.py new file mode 100644 index 000000000..14c74f802 --- /dev/null +++ b/tests/models/test_annotation_event_view.py @@ -0,0 +1,147 @@ +# ruff: noqa: E402 +"""Tests for the v_current_annotation_events view (current event per subject + type).""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.annotation_event_view import CurrentAnnotationEventView +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition + + +def _allele(session, digest, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + session.refresh(allele) + return allele + + +def _event(session, job_run, annotation_type, disposition, reason, *, allele_id=None, variant_id=None, **kw): + event = AnnotationEvent( + annotation_type=annotation_type, + allele_id=allele_id, + variant_id=variant_id, + disposition=disposition, + reason=reason, + job_run_id=job_run.id, + **kw, + ) + session.add(event) + session.commit() + session.refresh(event) + return event + + +def _view_rows(session, **filters): + stmt = select(CurrentAnnotationEventView) + for col, val in filters.items(): + stmt = stmt.where(getattr(CurrentAnnotationEventView, col) == val) + return list(session.scalars(stmt).all()) + + +def test_returns_only_latest_event_per_subject_and_type(session, job_run): + allele = _allele(session, "view-latest") + _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.ABSENT, "no_record", allele_id=allele.id + ) + latest = _event( + session, job_run, AnnotationType.GNOMAD_ALLELE_FREQUENCY, Disposition.PRESENT, "created", allele_id=allele.id + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].disposition == Disposition.PRESENT + + +def test_clinvar_is_multi_live_one_row_per_release(session, job_run): + allele = _allele(session, "view-clinvar") + e_2025 = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2025", + ) + _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="01_2026", + ) + e_2026_latest = _event( + session, + job_run, + AnnotationType.CLINVAR_CONTROL, + Disposition.PRESENT, + "superseded", + allele_id=allele.id, + source_version="01_2026", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.CLINVAR_CONTROL.value) + + by_version = {r.source_version: r.id for r in rows} + assert by_version == {"01_2025": e_2025.id, "01_2026": e_2026_latest.id} + + +def test_non_clinvar_collapses_across_versions(session, job_run): + """gnomAD/VEP supersede to a single current state: a re-fetch at a new source_version yields one + row, not one per version (the CASE folds source_version in only for ClinVar).""" + allele = _allele(session, "view-gnomad-version") + _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "created", + allele_id=allele.id, + source_version="4.0.0", + ) + latest = _event( + session, + job_run, + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + Disposition.PRESENT, + "reconfirmed", + allele_id=allele.id, + source_version="4.1.0", + ) + + rows = _view_rows(session, allele_id=allele.id, annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].source_version == "4.1.0" + + +def test_variant_subject_events_present_and_keyed_by_variant(session, setup_lib_db_with_variant, job_run): + variant = setup_lib_db_with_variant + _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.FAILED, "failed", variant_id=variant.id) + latest = _event(session, job_run, AnnotationType.VRS_MAPPING, Disposition.PRESENT, "mapped", variant_id=variant.id) + + rows = _view_rows(session, variant_id=variant.id, annotation_type=AnnotationType.VRS_MAPPING.value) + + assert [r.id for r in rows] == [latest.id] + assert rows[0].allele_id is None + + +def test_distinct_subjects_yield_distinct_rows(session, job_run): + a1 = _allele(session, "view-a1") + a2 = _allele(session, "view-a2") + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.PRESENT, "created", allele_id=a1.id) + _event(session, job_run, AnnotationType.CLINGEN_ALLELE_ID, Disposition.NOT_APPLICABLE, "no_hgvs", allele_id=a2.id) + + rows = _view_rows(session, annotation_type=AnnotationType.CLINGEN_ALLELE_ID.value) + + assert {r.allele_id for r in rows} == {a1.id, a2.id} From 5773aa5a6df132d5d1b0ed3f539a22da435bbf17 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:07 -0700 Subject: [PATCH 31/93] refactor(alleles): collapse to one work-unit per allele group_alleles_for_annotation now returns one job-specific payload per allele instead of an AlleleAnnotationGroup. Drops the authoritative_variant_ids bandaid seam (and is_authoritative from ScoreSetAlleleRow): annotation is an allele-level fact, and with the AnnotationEvent log the per-variant status fan-out goes away. --- src/mavedb/lib/clingen/alleles.py | 63 ++++++++++--------------------- tests/lib/clingen/test_alleles.py | 44 +++++++-------------- 2 files changed, 34 insertions(+), 73 deletions(-) diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index a2a962765..58cb6d7d0 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -5,8 +5,7 @@ score set. A single definition here prevents the two jobs from drifting apart. """ -from dataclasses import dataclass, field -from typing import Callable, Generic, Iterable, NamedTuple, Optional, TypeVar +from typing import Callable, Iterable, NamedTuple, Optional, TypeVar from sqlalchemy import select from sqlalchemy.orm import Session @@ -20,11 +19,9 @@ class ScoreSetAlleleRow(NamedTuple): - """One (allele, variant) link for a score set. An allele shared by multiple variants - appears once per variant so callers can fan annotation statuses out correctly. - - ``is_authoritative`` is a property of the link, not the allele: the same VRS allele can be - the authoritative measurement for one variant and an RT-derived equivalence for another. + """One (allele, variant) link for a score set. An allele shared by multiple variants appears once + per variant; :func:`group_alleles_for_annotation` collapses those duplicates into one work-unit + per allele. ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the VEP job can build its HGVS payload without a second query. They are optional with a ``None`` @@ -35,7 +32,6 @@ class ScoreSetAlleleRow(NamedTuple): post_mapped: dict | None clingen_allele_id: str | None variant_id: int - is_authoritative: bool hgvs_g: str | None = None hgvs_c: str | None = None hgvs_p: str | None = None @@ -57,7 +53,6 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.post_mapped, Allele.clingen_allele_id, Variant.id.label("variant_id"), - MappingRecordAllele.is_authoritative, Allele.hgvs_g, Allele.hgvs_c, Allele.hgvs_p, @@ -72,41 +67,21 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ - ScoreSetAlleleRow( - r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.is_authoritative, r.hgvs_g, r.hgvs_c, r.hgvs_p - ) + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p) for r in rows ] -@dataclass -class AlleleAnnotationGroup(Generic[P]): - """One allele's annotation work-unit: a job-specific ``payload`` plus the variants that receive a - per-variant annotation-status (VAS) row for it. - - ``authoritative_variant_ids`` is the interim **bandaid seam** (see - docs/design/allele-annotation-status.md). An annotation is an allele-level fact, but VAS is - per-variant; fanning it to every variant the allele backs would write multiple ``current`` rows - for one variant. So status is fanned only to the variants for which this allele is the - *authoritative* measurement. At the AnnotationEvent migration the per-variant fan-out goes away - and this list is no longer needed. The allele is still grouped (and linked/annotated at the - allele level) regardless — a purely RT-derived allele simply has an empty list here. - """ - - payload: P - authoritative_variant_ids: list[int] = field(default_factory=list) - - def group_alleles_for_annotation( rows: Iterable[ScoreSetAlleleRow], payload: Callable[[ScoreSetAlleleRow], Optional[P]], -) -> dict[int, AlleleAnnotationGroup[P]]: +) -> dict[int, P]: """Collapse the per-(allele, variant) rows from :func:`get_alleles_for_score_set` into one - work-unit per allele, keyed by ``allele_id``. + job-specific payload per allele, keyed by ``allele_id``. - The same allele recurs once per variant that links it (and can be authoritative for one variant - while RT-derived for another). This groups those rows so every annotation job shares one shape: - one entry per allele, carrying the authoritative-variant fan-out set for the bandaid. + The same allele recurs once per variant that links it, so this dedups those rows down to one + entry per allele — the shape every allele-keyed annotation job wants now that annotation events + are allele-keyed (one event per allele, never fanned per-variant). ``payload`` builds the job-specific payload from the first row seen for an allele — the CAID for gnomAD/ClinVar, the HGVS for VEP, etc. Returning ``None`` skips the allele entirely (e.g. it @@ -118,13 +93,15 @@ def group_alleles_for_annotation( is the permanent, never-reused surrogate. If annotation storage later keys on the digest, carry it on the row and store against it — the grouping contract here does not change. """ - groups: dict[int, AlleleAnnotationGroup[P]] = {} + groups: dict[int, P] = {} for row in rows: - if row.allele_id not in groups: - built = payload(row) - if built is None: - continue - groups[row.allele_id] = AlleleAnnotationGroup(payload=built) - if row.is_authoritative: - groups[row.allele_id].authoritative_variant_ids.append(row.variant_id) + if row.allele_id in groups: + continue + + built = payload(row) + if built is None: + continue + + groups[row.allele_id] = built + return groups diff --git a/tests/lib/clingen/test_alleles.py b/tests/lib/clingen/test_alleles.py index fbcfaa986..440646244 100644 --- a/tests/lib/clingen/test_alleles.py +++ b/tests/lib/clingen/test_alleles.py @@ -3,66 +3,50 @@ from mavedb.lib.clingen.alleles import ScoreSetAlleleRow, group_alleles_for_annotation -def _row(allele_id, variant_id, *, is_authoritative, caid="CA1"): +def _row(allele_id, variant_id, *, caid="CA1"): return ScoreSetAlleleRow( allele_id=allele_id, post_mapped={"type": "Allele"}, clingen_allele_id=caid, variant_id=variant_id, - is_authoritative=is_authoritative, ) -def test_collapses_rows_to_one_group_per_allele(): +def test_collapses_rows_to_one_payload_per_allele(): rows = [ - _row(1, 10, is_authoritative=True), - _row(1, 11, is_authoritative=True), - _row(2, 12, is_authoritative=True, caid="CA2"), + _row(1, 10), + _row(1, 11), + _row(2, 12, caid="CA2"), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert set(groups) == {1, 2} - assert groups[1].payload == "CA1" - assert groups[1].authoritative_variant_ids == [10, 11] - assert groups[2].authoritative_variant_ids == [12] + assert groups == {1: "CA1", 2: "CA2"} -def test_only_authoritative_links_contribute_variant_ids(): - """An allele that is authoritative for one variant and RT-derived for another fans status only to - the authoritative variant — the bandaid invariant.""" +def test_shared_allele_yields_a_single_entry(): + """An allele linked by multiple variants is deduped to one work-unit (events are allele-keyed).""" rows = [ - _row(1, 10, is_authoritative=True), - _row(1, 11, is_authoritative=False), + _row(1, 10), + _row(1, 11), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert groups[1].authoritative_variant_ids == [10] - - -def test_purely_rt_derived_allele_is_grouped_with_empty_fan_out(): - """A non-authoritative-only allele is still grouped (so it gets linked/annotated at the allele - level) but contributes no per-variant VAS row.""" - rows = [_row(1, 10, is_authoritative=False)] - - groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - - assert set(groups) == {1} - assert groups[1].authoritative_variant_ids == [] + assert groups == {1: "CA1"} def test_payload_returning_none_skips_the_allele(): """Returning None from payload drops the allele entirely — replacing each job's ad-hoc 'no CAID / no HGVS -> continue' filter.""" rows = [ - _row(1, 10, is_authoritative=True, caid=None), - _row(2, 11, is_authoritative=True, caid="CA2"), + _row(1, 10, caid=None), + _row(2, 11, caid="CA2"), ] groups = group_alleles_for_annotation(rows, payload=lambda r: r.clingen_allele_id) - assert set(groups) == {2} + assert groups == {2: "CA2"} def test_empty_rows_yield_empty_grouping(): From 5d35ed095fb3f2d491cb1325f4f0369b0033949f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:22 -0700 Subject: [PATCH 32/93] refactor(annotation): rewrite AnnotationStatusManager to emit events AnnotationStatusManager now writes AnnotationEvent rows via record_event and derives current status from the event log instead of maintaining per-variant status rows. Resolves subjects through VARIANT_SUBJECT_TYPES/ALLELE_SUBJECT_TYPES and the Disposition vocabulary. --- src/mavedb/lib/annotation_status_manager.py | 253 ++-- tests/lib/test_annotation_status_manager.py | 1456 ++----------------- 2 files changed, 242 insertions(+), 1467 deletions(-) diff --git a/src/mavedb/lib/annotation_status_manager.py b/src/mavedb/lib/annotation_status_manager.py index 6598bfabd..4022cb4d2 100644 --- a/src/mavedb/lib/annotation_status_manager.py +++ b/src/mavedb/lib/annotation_status_manager.py @@ -1,207 +1,170 @@ -"""Manage annotation statuses for variants. - -This module provides functionality to insert and retrieve annotation statuses -for genetic variants, ensuring that only one current status exists per -(variant, annotation type, version) combination. +"""Append-only writer for the annotation event log. + +Buffers :class:`AnnotationEvent` rows and flushes them in batches. This is an +**append-only** log, not a state table: there is no ``current`` flag to +maintain and no retire-on-write step. "Current" is derived at read time +(``DISTINCT ON (subject, annotation_type) … ORDER BY id DESC``) — exposed as +the ``v_current_annotation_events`` view (``mavedb.models.annotation_event_view``). + +Each event's *subject* is either a variant or an allele, chosen by +``annotation_type``. The writer validates the subject/type pairing up front so +a mis-subjected event fails with a clear ``ValueError`` rather than a deferred +DB ``CHECK`` violation at flush. """ import logging from typing import Optional -from sqlalchemy import select, update +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.sql import desc from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.annotation_event import ALLELE_SUBJECT_TYPES, VARIANT_SUBJECT_TYPES, AnnotationEvent logger = logging.getLogger(__name__) -# Default number of pending annotations to accumulate before auto-flushing. +# Default number of pending events to accumulate before auto-flushing. DEFAULT_BATCH_SIZE = 500 class AnnotationStatusManager: - """ - Manager for handling variant annotation statuses with batched writes. + """Buffered, append-only writer for :class:`AnnotationEvent` rows. - Annotations are accumulated in memory and flushed to the database in - batches (default 500) to reduce round-trips. Callers **must** call - :meth:`flush` after the last ``add_annotation`` to persist any remainder. + Events are accumulated in memory and flushed in batches (default 500) to + reduce round-trips. Callers **must** call :meth:`flush` after the last + :meth:`record_event` to persist any remainder. """ - def __init__(self, session: Session, job_run_id: Optional[int] = None, *, batch_size: int = DEFAULT_BATCH_SIZE): + def __init__( + self, + session: Session, + job_run_id: Optional[int] = None, + *, + score_set_id: Optional[int] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + ): self.session = session self.job_run_id = job_run_id + self.score_set_id = score_set_id self.batch_size = batch_size - self._pending: list[VariantAnnotationStatus] = [] - self._retirement_filters: list[dict] = [] + self._pending: list[AnnotationEvent] = [] - def add_annotation( + def record_event( self, - variant_id: int, annotation_type: AnnotationType, - status: AnnotationStatus, - version: Optional[str] = None, - failure_category: Optional[AnnotationFailureCategory] = None, - annotation_data: dict = {}, - current: bool = True, - replace_all_versions: bool = True, + *, + disposition: Disposition, + reason: str, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + metadata: Optional[dict] = None, + score_set_id: Optional[int] = None, ) -> None: - """ - Stage a new annotation and schedule retirement of previous current rows. - - By default (``replace_all_versions=True``), all existing current annotations for - (variant, type) are retired regardless of version. + """Buffer one terminal observation about a variant or an allele. - When ``replace_all_versions=False``, only existing current annotations matching - (variant, type, version) are retired. + Exactly one of ``variant_id`` / ``allele_id`` must be set, and it must + match the subject the ``annotation_type`` keys on. ``reason`` is the + single "what happened" axis across all dispositions (see EventReason). - Writes are accumulated in memory and flushed to the database when - ``batch_size`` is reached. Call :meth:`flush` after the last add to - persist any remaining annotations. - - NOTE: - This method does not commit the session. The caller is responsible - for persisting changes (e.g., via ``session.commit()``). + Writes are accumulated in memory and flushed when ``batch_size`` is + reached. Call :meth:`flush` after the last call to persist the + remainder. Does not commit — the caller owns the transaction. """ - self._retirement_filters.append( - { - "variant_id": variant_id, - "annotation_type": annotation_type, - "replace_all_versions": replace_all_versions, - "version": version, - } - ) + self._validate_subject(annotation_type, variant_id, allele_id) self._pending.append( - VariantAnnotationStatus( - variant_id=variant_id, + AnnotationEvent( annotation_type=annotation_type, - status=status, - version=version, - failure_category=failure_category, - current=current, + variant_id=variant_id, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + event_metadata=metadata, job_run_id=self.job_run_id, - **annotation_data, + score_set_id=score_set_id if score_set_id is not None else self.score_set_id, ) # type: ignore[call-arg] ) if len(self._pending) >= self.batch_size: self.flush() - def flush(self) -> None: - """Flush all pending annotations to the database. + @staticmethod + def _validate_subject(annotation_type: AnnotationType, variant_id: Optional[int], allele_id: Optional[int]) -> None: + if (variant_id is None) == (allele_id is None): + raise ValueError("Exactly one of variant_id or allele_id must be set") - Retires old ``current=True`` rows in bulk, then inserts all pending - new rows in a single ``add_all`` + ``flush``. This replaces the - previous pattern of 2 flushes per ``add_annotation`` call. - """ + type_value = annotation_type.value if isinstance(annotation_type, AnnotationType) else annotation_type + if variant_id is not None and type_value not in VARIANT_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is allele-subject; pass allele_id, not variant_id") + if allele_id is not None and type_value not in ALLELE_SUBJECT_TYPES: + raise ValueError(f"annotation_type {type_value!r} is variant-subject; pass variant_id, not allele_id") + + def flush(self) -> None: + """Insert all pending events in a single ``add_all`` + ``flush``.""" if not self._pending: return - self._retire_existing() self.session.add_all(self._pending) self.session.flush() - logger.debug(f"Flushed {len(self._pending)} annotation statuses") + logger.debug(f"Flushed {len(self._pending)} variant events") self._pending.clear() - self._retirement_filters.clear() - - def _retire_existing(self) -> None: - """Bulk-retire existing current annotations for all pending writes. - - Groups retirement filters by (annotation_type, replace_all_versions, version) - and issues one UPDATE per group, minimizing round-trips. - """ - # Group filters to minimize UPDATE statements. - # Key: (annotation_type, replace_all_versions, version) -> list of variant_ids - groups: dict[tuple, list[int]] = {} - for f in self._retirement_filters: - key = (f["annotation_type"], f["replace_all_versions"], f["version"]) - groups.setdefault(key, []).append(f["variant_id"]) - - for (annotation_type, replace_all_versions, version), variant_ids in groups.items(): - conditions = [ - VariantAnnotationStatus.variant_id.in_(variant_ids), - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ] - if not replace_all_versions: - conditions.append(VariantAnnotationStatus.version == version) - - stmt = update(VariantAnnotationStatus).where(*conditions).values(current=False) - self.session.execute(stmt) def get_current_annotation( - self, variant_id: int, annotation_type: AnnotationType, version: Optional[str] = None - ) -> Optional[VariantAnnotationStatus]: - """ - Retrieve the current annotation for a given variant/type/version. - - Flushes pending annotations first to ensure the result is up to date. - """ - self.flush() - - stmt = select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - VariantAnnotationStatus.current.is_(True), - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) - - result = self.session.execute(stmt) - return result.scalar_one_or_none() - - def get_annotation_history( self, - variant_id: int, annotation_type: AnnotationType, - version: Optional[str] = None, - ) -> list[VariantAnnotationStatus]: - """ - Return the full annotation timeline for a variant/type, newest first. - - Includes both current and retired rows — useful for debugging and - support investigations. + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> Optional[AnnotationEvent]: + """Latest event for a single ``(subject, annotation_type)`` key. + + Current status is the newest event by ``id`` — there is no ``current`` + flag to filter on. Flushes pending events first so the result reflects + buffered writes. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.annotation_type == annotation_type, - ) - .order_by(desc(VariantAnnotationStatus.id)) - ) - - if version is not None: - stmt = stmt.where(VariantAnnotationStatus.version == version) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) - return list(self.session.scalars(stmt).all()) + stmt = stmt.order_by(desc(AnnotationEvent.id)).limit(1) + return self.session.scalars(stmt).first() - def get_all_current_annotations( + def get_event_history( self, - variant_id: int, - ) -> list[VariantAnnotationStatus]: - """ - Return all current annotations for a variant, across all types and versions. - - Useful for a quick overview of what annotations are active for a given variant. + annotation_type: AnnotationType, + *, + variant_id: Optional[int] = None, + allele_id: Optional[int] = None, + source_version: Optional[str] = None, + ) -> list[AnnotationEvent]: + """Full event timeline for a ``(subject, annotation_type)`` key, newest first. + + The append-only log retains every observation — skips, reconfirms, + no-ops — so this is the complete audit trail, not just the current row. """ self.flush() + self._validate_subject(annotation_type, variant_id, allele_id) - stmt = ( - select(VariantAnnotationStatus) - .where( - VariantAnnotationStatus.variant_id == variant_id, - VariantAnnotationStatus.current.is_(True), - ) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.version) - ) + stmt = select(AnnotationEvent).where(AnnotationEvent.annotation_type == annotation_type) + if variant_id is not None: + stmt = stmt.where(AnnotationEvent.variant_id == variant_id) + else: + stmt = stmt.where(AnnotationEvent.allele_id == allele_id) + if source_version is not None: + stmt = stmt.where(AnnotationEvent.source_version == source_version) + stmt = stmt.order_by(desc(AnnotationEvent.id)) return list(self.session.scalars(stmt).all()) diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index 52771b6bf..b616fbf34 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -5,1362 +5,174 @@ pytest.importorskip("psycopg2") from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.models.allele import Allele from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus -from mavedb.models.variant import Variant +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason @pytest.fixture def annotation_status_manager(session, job_run): - """Fixture to provide an AnnotationStatusManager instance.""" return AnnotationStatusManager(session, job_run_id=job_run.id) @pytest.fixture -def existing_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() +def allele(session): + allele = Allele(vrs_digest="asm-test-allele-digest", level="genomic") + session.add(allele) session.commit() + session.refresh(allele) + return allele - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.fixture -def existing_unversioned_annotation_status(session, annotation_status_manager, setup_lib_db_with_variant): - """Fixture to create an existing annotation status in the database.""" - - # Add initial annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert annotation.id is not None - assert annotation.current is True - - return annotation - - -@pytest.mark.unit -class TestAnnotationStatusManagerCreateAnnotationUnit: - """Unit tests for AnnotationStatusManager.add_annotation method.""" - - @pytest.mark.parametrize( - "annotation_type", - AnnotationType._member_map_.values(), - ) - @pytest.mark.parametrize( - "status", - AnnotationStatus._member_map_.values(), - ) - def test_add_annotation_creates_entry_with_annotation_type_version_status( - self, session, annotation_status_manager, annotation_type, status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry with correct type and version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - annotation_data={}, - current=True, - status=status, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=annotation_type, - version="v1.0", - ) - - assert annotation is not None - assert annotation.annotation_type == annotation_type - assert annotation.status == status - assert annotation.version == "v1.0" - - def test_add_annotation_stores_job_run_id( - self, session, annotation_status_manager, job_run, setup_lib_db_with_variant - ): - """Test that every annotation is created with the job_run_id from the manager.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - annotation_data={}, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.job_run_id == job_run.id - - def test_add_annotation_persists_annotation_data( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding an annotation persists the provided annotation data.""" - annotation_data = { - "annotation_metadata": {"some_key": "some_value"}, - "error_message": None, - } - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - status=AnnotationStatus.SUCCESS, - version="v1.0", - failure_category=None, - annotation_data=annotation_data, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1.0", - ) - - assert annotation is not None - assert annotation.failure_category is None - for key, value in annotation_data.items(): - assert getattr(annotation, key) == value - - def test_add_annotation_creates_entry_and_marks_previous_not_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation creates a new entry and marks previous ones as not current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type, version) - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - def test_add_annotation_with_different_version_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different version keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same (variant, type) but different version - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_with_different_type_keeps_previous_current( - self, session, job_run, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation with a different type keeps previous current.""" - manager = AnnotationStatusManager(session, job_run_id=job_run.id) - - # Add second annotation for same variant but different type - manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - manager.flush() - session.commit() - - annotation = manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_add_annotation_without_version(self, session, annotation_status_manager, setup_lib_db_with_variant): - """Test that adding an annotation without specifying version works correctly.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - version=None, - annotation_data={}, - status=AnnotationStatus.SKIPPED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - ) - - assert annotation is not None - assert annotation.id is not None - assert annotation.version is None - assert annotation.current is True - - def test_add_annotation_multiple_without_version_marks_previous_not_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding multiple annotations without version marks previous ones as not current.""" - - # Add second annotation without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is False - - def test_add_annotation_different_type_without_version_keeps_previous_current( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that adding an annotation of different type without version keeps previous current.""" - - # Add second annotation of different type without version - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - second_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - ) - - assert second_annotation is not None - assert second_annotation.id is not None - assert second_annotation.current is True - - # Refresh first annotation from DB - session.refresh(existing_unversioned_annotation_status) - assert existing_unversioned_annotation_status.current is True - - def test_add_annotation_multiple_variants_independent_current_flags( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants maintains independent current flags.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert annotation1 is not None - assert annotation1.id is not None - assert annotation1.current is True - - assert annotation2 is not None - assert annotation2.id is not None - assert annotation2.current is True - - -class TestAnnotationStatusManagerGetCurrentAnnotationUnit: - """Unit tests for AnnotationStatusManager.get_current_annotation method.""" - - def test_get_current_annotation_returns_none_when_no_entry( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when no entry exists.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_returns_correct_entry( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation.id == existing_annotation_status.id - assert annotation.current is True - - def test_get_current_annotation_returns_none_for_non_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation returns None when the entry is not current.""" - # Mark existing annotation as not current - existing_annotation_status.current = False - session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_with_different_version_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different version returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert annotation is None - - def test_get_current_annotation_with_different_type_returns_none( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation with different type returns None.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version="v1", - ) - assert annotation is None - - def test_get_current_annotation_without_version_returns_correct_entry( - self, session, annotation_status_manager, existing_unversioned_annotation_status, setup_lib_db_with_variant - ): - """Test that getting current annotation without version returns the correct entry.""" - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=None, - ) - assert annotation.id == existing_unversioned_annotation_status.id - assert annotation.current is True - - -class TestAnnotationStatusManagerIntegration: - """Integration tests for AnnotationStatusManager methods.""" - - def test_add_and_get_current_annotation_work_together( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Test that adding and getting current annotation work together correctly.""" - # Add annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_multiple_and_get_current_returns_latest( - self, session, annotation_status_manager, version, setup_lib_db_with_variant - ): - """Test that adding multiple annotations and getting current returns the latest one.""" - # Add first annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Add second annotation - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation - retrieved_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation is not None - assert retrieved_annotation.current is True - assert retrieved_annotation.version == version - assert retrieved_annotation.status == AnnotationStatus.SUCCESS - - @pytest.mark.parametrize( - "version", - ["v1.0", "v2.0", None], - ) - def test_add_annotations_for_different_variants_and_get_current_independent( - self, session, annotation_status_manager, version, setup_lib_db_with_score_set - ): - """Test that adding annotations for different variants and getting current works independently.""" - - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - # Add annotation for variant 1 - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - - # Add annotation for variant 2 - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - # Get current annotation for variant 1 - retrieved_annotation1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation1 is not None - assert retrieved_annotation1.current is True - assert retrieved_annotation1.status == AnnotationStatus.SUCCESS - assert retrieved_annotation1.version == version - - # Get current annotation for variant 2 - retrieved_annotation2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version=version, - ) - - assert retrieved_annotation2 is not None - assert retrieved_annotation2.current is True - assert retrieved_annotation2.status == AnnotationStatus.FAILED - assert retrieved_annotation2.version == version - - -@pytest.mark.unit -class TestAnnotationStatusManagerReplaceAllVersionsUnit: - """Unit tests for the replace_all_versions parameter of AnnotationStatusManager.add_annotation.""" - - def test_replace_all_versions_false_keeps_different_version_current( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """Default behavior: a new annotation only retires the same version, not others.""" - # existing_annotation_status is version "v1", current=True - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_annotation is not None - assert new_annotation.current is True - - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is True - - def test_replace_all_versions_true_retires_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True retires all current records for (variant, type) regardless of version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - # Both v1 and v2 are current at this point (replace_all_versions=False) - v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert v1 is not None and v1.current is True - assert v2 is not None and v2.current is True - # Now add v3 with replace_all_versions=True — should retire both v1 and v2 - annotation_status_manager.add_annotation( +class TestRecordEvent: + def test_records_variant_subject_event(self, session, annotation_status_manager, setup_lib_db_with_variant): + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) annotation_status_manager.flush() session.commit() - session.refresh(v1) - session.refresh(v2) - assert v1.current is False - assert v2.current is False - - v3 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v3", + event = annotation_status_manager.get_current_annotation( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id ) - assert v3 is not None and v3.current is True + assert event is not None + assert event.disposition == Disposition.PRESENT + assert event.allele_id is None + # No current flag exists on the event log. + assert not hasattr(event, "current") - def test_replace_all_versions_true_only_affects_matching_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """replace_all_versions=True only retires records for the same annotation_type.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, + def test_records_allele_subject_event_with_metadata(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + metadata={"gnomad_variant_id": "1-55051215-G-A"}, ) annotation_status_manager.flush() session.commit() - vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # replace VRS_MAPPING only - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs) - session.refresh(clinvar) - assert vrs.current is False - assert clinvar.current is True + assert event.event_metadata == {"gnomad_variant_id": "1-55051215-G-A"} + assert event.source_version == "4.1.0" + assert event.variant_id is None - new_vrs = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_vrs is not None and new_vrs.current is True - def test_replace_all_versions_true_only_affects_matching_variant( - self, session, annotation_status_manager, setup_lib_db_with_score_set - ): - """replace_all_versions=True only retires records for the same variant_id.""" - variant1 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.1A>G", hgvs_pro="NP_000000.1:p.Met1Val", data={}) - variant2 = Variant(score_set_id=1, hgvs_nt="NM_000000.1:c.2A>T", hgvs_pro="NP_000000.1:p.Met2Val", data={}) - session.add_all([variant1, variant2]) - session.commit() - session.refresh(variant1) - session.refresh(variant2) - - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - ann2 = annotation_status_manager.get_current_annotation( - variant_id=variant2.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - - # replace variant1 only - annotation_status_manager.add_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(ann1) - session.refresh(ann2) - assert ann1.current is False - assert ann2.current is True # untouched - - new_ann1 = annotation_status_manager.get_current_annotation( - variant_id=variant1.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - assert new_ann1 is not None and new_ann1.current is True - - def test_replace_all_versions_true_same_version_also_retired( - self, session, annotation_status_manager, existing_annotation_status, setup_lib_db_with_variant - ): - """replace_all_versions=True retires a same-version record just as replace_all_versions=False would.""" - # existing_annotation_status is version "v1" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=True, - ) +class TestAppendOnlyLatestWins: + def test_latest_event_by_id_is_current(self, session, annotation_status_manager, allele): + for reason in (EventReason.CREATED, EventReason.RECONFIRMED, EventReason.SKIPPED): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=reason, + source_version="4.1.0", + ) annotation_status_manager.flush() session.commit() - session.refresh(existing_annotation_status) - assert existing_annotation_status.current is False - - new_annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + # All three events persist (append-only — no retire of prior rows). + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id ) - assert new_annotation is not None - assert new_annotation.current is True - assert new_annotation.status == AnnotationStatus.FAILED - - -@pytest.mark.unit -class TestAnnotationStatusManagerBatchingUnit: - """Unit tests for batching and flush behavior.""" - - def test_flush_noop_when_empty(self, annotation_status_manager): - """flush() with no pending annotations does nothing and does not error.""" - annotation_status_manager.flush() # should not raise - - def test_auto_flush_at_batch_size(self, session, setup_lib_db_with_score_set): - """Annotations are auto-flushed to the DB when batch_size is reached.""" - variants = [ - Variant(score_set_id=1, hgvs_nt=f"NM_000000.1:c.{i}A>G", hgvs_pro=f"NP_000000.1:p.Met{i}Val", data={}) - for i in range(3) + assert [e.reason for e in history] == [ # newest first + EventReason.SKIPPED, + EventReason.RECONFIRMED, + EventReason.CREATED, ] - session.add_all(variants) - session.commit() - for v in variants: - session.refresh(v) - - manager = AnnotationStatusManager(session, batch_size=2) - - # Add first — stays pending (below threshold) - manager.add_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Add second — triggers auto-flush (reaches batch_size=2) - manager.add_annotation( - variant_id=variants[1].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 0 # flushed - - # Verify the auto-flushed rows are visible in the DB - ann = manager.get_current_annotation( - variant_id=variants[0].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann is not None and ann.current is True - - # Add a third — stays pending (below threshold again) - manager.add_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - assert len(manager._pending) == 1 - - # Explicit flush persists the remainder - manager.flush() - assert len(manager._pending) == 0 - - ann3 = manager.get_current_annotation( - variant_id=variants[2].id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert ann3 is not None and ann3.current is True - - def test_get_current_annotation_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_current_annotation() flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush — get_current_annotation should auto-flush - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - assert annotation is not None - assert annotation.current is True - assert len(annotation_status_manager._pending) == 0 - def test_flush_clears_internal_buffers(self, session, annotation_status_manager, setup_lib_db_with_variant): - """flush() clears both _pending and _retirement_filters.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) + # Current = newest by id. + current = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert current.reason == EventReason.SKIPPED + + def test_source_version_scopes_current(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.0.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + v40 = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id, source_version="4.0.0" + ) + assert v40.disposition == Disposition.PRESENT + + +class TestSubjectValidation: + def test_variant_subject_type_with_allele_id_raises(self, annotation_status_manager, allele): + with pytest.raises(ValueError, match="variant-subject"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_allele_subject_type_with_variant_id_raises(self, annotation_status_manager, setup_lib_db_with_variant): + with pytest.raises(ValueError, match="allele-subject"): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_neither_subject_raises(self, annotation_status_manager): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with_variant, allele): + with pytest.raises(ValueError, match="Exactly one"): + annotation_status_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=setup_lib_db_with_variant.id, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + + +class TestBatching: + def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): + annotation_status_manager.batch_size = 2 + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, + variant_id=setup_lib_db_with_variant.id, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + ) + # Not yet flushed (1 < batch_size). assert len(annotation_status_manager._pending) == 1 - assert len(annotation_status_manager._retirement_filters) == 1 - - annotation_status_manager.flush() - assert len(annotation_status_manager._pending) == 0 - assert len(annotation_status_manager._retirement_filters) == 0 - - def test_batch_retirement_groups_by_annotation_type( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """Multiple annotation types in one batch are retired independently.""" - # Create initial annotations for two types - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - vrs_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - clinvar_v1 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v1", - ) - - # Now add replacements for both types in one batch - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - session.refresh(vrs_v1) - session.refresh(clinvar_v1) - assert vrs_v1.current is False - assert clinvar_v1.current is False - - vrs_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - ) - clinvar_v2 = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="v2", - ) - assert vrs_v2 is not None and vrs_v2.current is True - assert clinvar_v2 is not None and clinvar_v2.current is True - - -@pytest.mark.unit -class TestAnnotationStatusManagerAuditHelpersUnit: - """Unit tests for audit query helpers: get_annotation_history and get_all_current_annotations.""" - - def test_get_annotation_history_returns_all_rows_newest_first( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history returns both current and retired rows, newest first.""" - # Create two annotations for the same (variant, type, version) — first gets retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( + annotation_status_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, ) - - assert len(history) == 2 - # Newest first - assert history[0].status == AnnotationStatus.FAILED - assert history[0].current is True - assert history[1].status == AnnotationStatus.SUCCESS - assert history[1].current is False - - def test_get_annotation_history_filters_by_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history with version only returns matching rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history_jan = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - ) - assert len(history_jan) == 1 - assert history_jan[0].version == "2025-01" - - def test_get_annotation_history_without_version_returns_all_versions( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history without version returns rows across all versions.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - ) - assert len(history) == 2 - - def test_get_annotation_history_empty_for_no_records(self, annotation_status_manager, setup_lib_db_with_variant): - """get_annotation_history returns empty list when no records exist.""" - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - assert history == [] - - def test_get_annotation_history_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_annotation_history flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - history = annotation_status_manager.get_annotation_history( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - ) - assert len(history) == 1 + # Auto-flushed at batch_size=2. assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_returns_all_types( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns current annotations across all types.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - types = {a.annotation_type for a in all_current} - assert types == { - AnnotationType.VRS_MAPPING, - AnnotationType.CLINVAR_CONTROL, - AnnotationType.CLINGEN_ALLELE_ID, - } - - def test_get_all_current_annotations_excludes_retired( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations does not include retired rows.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() - - # Replace it — v1 becomes retired - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v2", - annotation_data={}, - status=AnnotationStatus.FAILED, - current=True, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 1 - assert all_current[0].version == "v2" - - def test_get_all_current_annotations_empty_for_no_records( - self, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns empty list when no records exist.""" - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert result == [] - - def test_get_all_current_annotations_auto_flushes_pending( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations flushes pending writes before querying.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - # No explicit flush - result = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(result) == 1 - assert len(annotation_status_manager._pending) == 0 - - def test_get_all_current_annotations_ordered_by_type_then_version( - self, session, annotation_status_manager, setup_lib_db_with_variant - ): - """get_all_current_annotations returns results ordered by annotation_type, version.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-02", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version="2025-01", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - replace_all_versions=False, - ) - annotation_status_manager.flush() - session.commit() - - all_current = annotation_status_manager.get_all_current_annotations( - variant_id=setup_lib_db_with_variant.id, - ) - assert len(all_current) == 3 - # clinvar_control < vrs_mapping alphabetically - assert all_current[0].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[0].version == "2025-01" - assert all_current[1].annotation_type == AnnotationType.CLINVAR_CONTROL - assert all_current[1].version == "2025-02" - assert all_current[2].annotation_type == AnnotationType.VRS_MAPPING - - -@pytest.mark.unit -class TestVariantAnnotationStatusReprUnit: - """Unit tests for the VariantAnnotationStatus __repr__ method.""" - - def test_repr_includes_key_fields(self, session, annotation_status_manager, setup_lib_db_with_variant): - """__repr__ includes id, variant_id, type, version, status, current, and created_at.""" - annotation_status_manager.add_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - annotation_data={}, - status=AnnotationStatus.SUCCESS, - current=True, - ) - annotation_status_manager.flush() session.commit() - - annotation = annotation_status_manager.get_current_annotation( - variant_id=setup_lib_db_with_variant.id, - annotation_type=AnnotationType.VRS_MAPPING, - version="v1", - ) - repr_str = repr(annotation) - - assert "VariantAnnotationStatus" in repr_str - assert f"id={annotation.id}" in repr_str - assert f"variant_id={setup_lib_db_with_variant.id}" in repr_str - assert "type='vrs_mapping'" in repr_str - assert "version='v1'" in repr_str - assert "status='success'" in repr_str - assert "current=True" in repr_str - assert "created_at=" in repr_str From aacc13aadcaaff73dff115b2a7ffe05026679961 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:31 -0700 Subject: [PATCH 33/93] feat(gnomad): derive annotation events from link verdicts link_gnomad_variants_to_alleles now returns a GnomadLinkVerdict per touched allele (CREATED/UNCHANGED) as the single source of truth for per-allele status; the job derives annotation events directly from the verdict map rather than re-querying link state. Alleles absent from the map are read as 'gnomAD had no record'. --- src/mavedb/lib/gnomad.py | 38 ++++-- .../worker/jobs/external_services/gnomad.py | 119 +++++++++--------- tests/lib/test_gnomad.py | 41 +++--- .../jobs/external_services/test_gnomad.py | 108 +++++++++------- 4 files changed, 181 insertions(+), 125 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 8456ec35d..5fd4d888c 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,6 +1,7 @@ import logging import os import re +from enum import Enum from typing import Any, Sequence, Union from sqlalchemy import Connection, Row, func, select, text @@ -12,6 +13,9 @@ from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant +logger = logging.getLogger(__name__) + + GNOMAD_DB_NAME = "gnomAD" GNOMAD_DATA_VERSION = os.getenv("GNOMAD_DATA_VERSION", "v4.1") _CAID_LEADING_ZERO_RE = r"^(CA)0+([0-9])" @@ -21,7 +25,17 @@ in link_gnomad_variants_to_alleles. """ -logger = logging.getLogger(__name__) + +class GnomadLinkVerdict(str, Enum): + """Per-allele outcome of a gnomAD linking run, returned for every allele the linker touched. + + The single source of truth for what happened to an allele's link this run — the caller derives + annotation status directly from this, never by re-querying link state (which would be a second, + drift-prone source of truth). + """ + + CREATED = "created" # link created or superseded this run (a new/changed live link) + UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: @@ -150,7 +164,9 @@ def gnomad_variant_data_for_caids( return result_rows -def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[Row[Any]]) -> set[int]: +def link_gnomad_variants_to_alleles( + db: Session, gnomad_variant_data: Sequence[Row[Any]] +) -> dict[int, GnomadLinkVerdict]: """Link gnomAD variants to deduplicated alleles by CAID, superseding only on change. Every ``Allele`` carrying the row's ``clingen_allele_id`` (populated by CAR) is linked through a @@ -163,12 +179,16 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R it. A current-version link to a *different* identifier (a CAID re-resolved within one release) is logged and superseded newest-wins, not raised. - Does not commit. Returns the allele ids whose link was created or superseded this run. + Does not commit. Returns a verdict per allele *touched* this run (matched a CAID-bearing row): + :attr:`GnomadLinkVerdict.CREATED` for a created/superseded link, :attr:`~GnomadLinkVerdict.UNCHANGED` + for a live link left in place. Alleles absent from the map were matched by no row — the caller reads + those as "gnomAD had no record". This is the single source of truth for per-allele status; callers + must not re-derive it by re-querying link state. """ save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) - changed_allele_ids: set[int] = set() + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() @@ -241,6 +261,7 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R # No change: live link already points here — leave it untouched (no spurious boundary). if live_link is not None and live_link.gnomad_variant_id == gnomad_variant.id: + verdicts.setdefault(allele.id, GnomadLinkVerdict.UNCHANGED) continue if ( @@ -265,7 +286,7 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R [GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gnomad_variant.id)], GnomadAlleleLink.allele_id == allele.id, ) - changed_allele_ids.add(allele.id) + verdicts[allele.id] = GnomadLinkVerdict.CREATED # created always wins over a same-run unchanged logger.debug( msg=f"Linked gnomAD variant {gnomad_variant.db_identifier} to allele {allele.id} ({allele.clingen_allele_id})", @@ -276,9 +297,10 @@ def link_gnomad_variants_to_alleles(db: Session, gnomad_variant_data: Sequence[R f"Processed {len(alleles_with_caid)} alleles with CAID {row.caid} for gnomAD variant {gnomad_identifier_for_variant}. ({index}/{len(gnomad_variant_data)})" ) - save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + changed_allele_count = sum(1 for v in verdicts.values() if v is GnomadLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Created or superseded {len(changed_allele_ids)} allele links this run.", + msg=f"Created or superseded {changed_allele_count} allele links this run.", extra=logging_context(), ) - return changed_allele_ids + return verdicts diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 8daa5c9af..3b7a48d16 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -7,6 +7,7 @@ """ import logging +from collections import Counter from sqlalchemy import select @@ -15,12 +16,14 @@ from mavedb.lib.clingen.alleles import get_alleles_for_score_set, group_alleles_for_annotation from mavedb.lib.gnomad import ( GNOMAD_DATA_VERSION, + GnomadLinkVerdict, gnomad_variant_data_for_caids, link_gnomad_variants_to_alleles, ) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet @@ -33,34 +36,31 @@ def _annotate_gnomad( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a GNOMAD_ALLELE_FREQUENCY annotation out to every variant served by an allele. + """Record one GNOMAD_ALLELE_FREQUENCY event for an allele (frequency is an allele-level fact). - AAS migration seam: the single choke point for gnomAD's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (which variant's mapping drove the linkage). See - docs/design/allele-annotation-status.md. + The single choke point for gnomAD's status writes. One event per allele, stamped at the current + gnomAD version; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.GNOMAD_ALLELE_FREQUENCY, - version=GNOMAD_DATA_VERSION, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=GNOMAD_DATA_VERSION, + metadata=meta or None, + ) @with_pipeline_management @@ -113,12 +113,20 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the - # RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # RT-derived one. Events are allele-keyed, so every linked allele is recorded. allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=lambda row: row.clingen_allele_id, ) + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "skipped_allele_count": 0, + } + ) + num_alleles_with_caids = len(allele_data) job_manager.save_to_context({"num_alleles_to_link_gnomad": num_alleles_with_caids}) @@ -128,9 +136,7 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} - ) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) job_manager.update_progress( 10, 100, f"Found {num_alleles_with_caids} alleles with CAIDs to link to gnomAD variants." @@ -150,10 +156,11 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ).all() ) - # Cost: skip alleles already linked at the current version (they can't change). force re-fetches + # Skip alleles already linked at the current version (they can't change). force re-fetches # all; the linker still supersedes only on change, so a forced no-op writes nothing. already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) - variant_caids = sorted({allele_data[aid].payload for aid in allele_data if aid not in already_current}) + variant_caids = sorted({allele_data[aid] for aid in allele_data if aid not in already_current}) + job_manager.save_to_context( { "num_alleles_already_current": len(already_current), @@ -162,7 +169,7 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: } ) - changed_allele_ids: set[int] = set() + verdicts: dict[int, GnomadLinkVerdict] = {} if variant_caids: with athena.engine.connect() as athena_session: logger.debug("Fetching gnomAD variants from Athena.") @@ -175,7 +182,7 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ) logger.info(msg="Attempting to link alleles to gnomAD variants.", extra=job_manager.logging_context()) - changed_allele_ids = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) + verdicts = link_gnomad_variants_to_alleles(job_manager.db, gnomad_variant_data) job_manager.db.flush() else: logger.info( @@ -184,43 +191,43 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: ) job_manager.update_progress(75, 100, "All alleles already current at this gnomAD version.") - # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting - # if already current, SKIPPED if no current-version link (gnomAD had no data for the CAID). - current_after = alleles_linked_at_current_version(set(allele_data.keys())) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - created_count = preexisting_count = skipped_count = 0 - for allele_id, entry in allele_data.items(): - if allele_id in current_after: - action = "created" if allele_id in changed_allele_ids else "preexisting" - if action == "created": - created_count += 1 - else: - preexisting_count += 1 + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): + verdict = verdicts.get(allele_id) + if verdict is GnomadLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + metadata={"clingen_allele_id": caid}, + ) + elif allele_id in already_current or verdict is GnomadLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 _annotate_gnomad( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.payload, "action": action}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": caid}, ) else: - skipped_count += 1 + annotation_counts["skipped_allele_count"] += 1 _annotate_gnomad( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, error_message="No gnomAD variant could be linked for this allele.", - metadata={"clingen_allele_id": entry.payload}, + metadata={"clingen_allele_id": caid}, ) annotation_manager.flush() - outcome_data = { - "created_allele_count": created_count, - "preexisting_allele_count": preexisting_count, - "skipped_allele_count": skipped_count, - } + outcome_data = dict(annotation_counts) job_manager.save_to_context(outcome_data) logger.info(msg="Done linking gnomAD variants to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index e00b57e76..f9c8b17aa 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -9,6 +9,7 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.gnomad import ( + GnomadLinkVerdict, allele_list_from_list_like_string, gnomad_identifier, gnomad_table_name, @@ -170,7 +171,7 @@ def test_links_new_gnomad_variant_to_allele(session, mocked_gnomad_variant_row): with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() live_links = _live_links_for(session, allele.id) @@ -186,7 +187,7 @@ def test_can_link_gnomad_variants_with_none_type_faf_fields(session, mocked_gnom with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() live_links = _live_links_for(session, allele.id) @@ -202,7 +203,7 @@ def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() # Reused the existing gnomAD variant rather than creating a second. @@ -214,14 +215,20 @@ def test_links_existing_gnomad_variant(session, mocked_gnomad_variant_row): def test_re_running_unchanged_data_is_idempotent(session, mocked_gnomad_variant_row): """Supersede only on change: a second run with identical data writes nothing — one live link, - no retired rows, so the valid-time history records no spurious boundary.""" + no retired rows, so the valid-time history records no spurious boundary. The second run still + *reports* the allele (verdict UNCHANGED), so the caller can mark it preexisting without re-querying + link state.""" allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() - # Second run sees the live link already points to this gnomAD variant → no change reported. - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == set() + # Second run sees the live link already points to this gnomAD variant → unchanged, no DB write. + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.UNCHANGED + } session.commit() # One link, still live, never retired — the re-run did not churn the history. @@ -237,11 +244,15 @@ def test_version_bump_supersedes_to_single_live_link(session, mocked_gnomad_vari allele = _make_allele(session, mocked_gnomad_variant_row.caid, vrs_digest="vrs-1") with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v1.old"): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", "v2.new"): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() live_links = _live_links_for(session, allele.id) @@ -273,7 +284,9 @@ def test_same_version_different_identifier_supersedes_newest_wins(session, mocke session.commit() with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): - assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == {allele.id} + assert link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) == { + allele.id: GnomadLinkVerdict.CREATED + } session.commit() live_links = _live_links_for(session, allele.id) @@ -288,7 +301,7 @@ def test_links_one_gnomad_variant_to_multiple_alleles_sharing_a_caid(session, mo with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele1.id, allele2.id} + assert result == {allele1.id: GnomadLinkVerdict.CREATED, allele2.id: GnomadLinkVerdict.CREATED} session.commit() for allele in (allele1, allele2): @@ -305,15 +318,15 @@ def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gn with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == {allele.id} + assert result == {allele.id: GnomadLinkVerdict.CREATED} session.commit() assert len(_live_links_for(session, allele.id)) == 1 -def test_returns_empty_set_when_no_alleles_match(session, mocked_gnomad_variant_row): +def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) - assert result == set() + assert result == {} assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # No gnomAD variant is created when nothing matches the CAID. assert len(session.scalars(select(GnomADVariant)).all()) == 0 diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index 23316adb8..0b14fda21 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -14,7 +14,7 @@ from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.gnomad import link_gnomad_variants from mavedb.worker.lib.managers.job_manager import JobManager @@ -91,7 +91,7 @@ async def test_link_gnomad_variants_call_linking_method( ), patch( "mavedb.worker.jobs.external_services.gnomad.link_gnomad_variants_to_alleles", - return_value=set(), + return_value={}, ) as mock_linking_method, patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine), ): @@ -156,8 +156,8 @@ async def test_link_gnomad_variants_no_alleles_with_caids( assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered (since there were no alleles with CAIDs) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -189,11 +189,13 @@ async def test_link_gnomad_variants_no_matching_caids( # Verify that no allele links were created assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 - # Verify a skipped annotation status was rendered (since there was an allele with a CAID) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # Verify an absent event was rendered: the allele's CAID was queried but gnomAD had no record. + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id == allele.id and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) @@ -230,16 +232,18 @@ async def test_link_gnomad_variants_successful_linking_independent( assert len(live_links) == 1 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run) assert sample_link_gnomad_variants_run.status == JobStatus.SUCCEEDED - async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_link_gnomad_variants_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, @@ -249,11 +253,12 @@ async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_a setup_rt_derived_allele_with_caid, athena_engine, ): - """gnomAD linkage must cover the full allele set, not just authoritative links: the - RT-derived allele carries the CAID gnomAD matches and must be linked. Per-variant VAS status, - however, is written only for the authoritative link (the interim bandaid) — so exactly one - annotation row is produced, keyed to the variant, never a second row for the RT-derived - allele.""" + """gnomAD linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the CAID gnomAD matches and is linked. Events are now allele-keyed, so the + RT-derived allele's PRESENT status is recorded directly — the limitation the per-variant + bandaid had (dropping the RT-derived allele's status) is lifted. Each allele in the score set + gets its own event: present for the linked RT allele, absent for the unmatched authoritative + one.""" variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid with patch("mavedb.worker.jobs.external_services.gnomad.athena.engine", athena_engine): @@ -279,13 +284,16 @@ async def test_link_gnomad_variants_links_rt_derived_allele_but_annotates_only_a == 0 ) - # Annotation status is written only for the authoritative link: exactly one VAS row, for the - # variant. (Its status is "skipped" because the variant's authoritative allele had no match — - # cross-level resolution onto the RT-derived allele is deferred to the AnnotationEvent design.) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].variant_id == variant.id - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, + # the unmatched authoritative allele is absent. The variant resolves its derived allele's + # status through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "gnomad_allele_frequency" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" async def test_link_gnomad_variants_skips_allele_already_current( self, @@ -326,11 +334,11 @@ async def test_link_gnomad_variants_skips_allele_already_current( assert len(links) == 1 assert links[0].valid_to is None - # Status is SUCCESS, marked preexisting. - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + # Event is PRESENT, marked preexisting (the link was already current; not created this run). + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" async def test_link_gnomad_variants_force_refetches_without_churn( self, @@ -403,10 +411,12 @@ async def test_link_gnomad_variants_successful_linking_pipeline( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify job status updates session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -481,10 +491,12 @@ async def test_link_gnomad_variants_with_arq_context_independent( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run) @@ -514,10 +526,12 @@ async def test_link_gnomad_variants_with_arq_context_pipeline( assert len(session.scalars(select(GnomadAlleleLink).where(GnomadAlleleLink.current)).all()) > 0 # Verify annotation status was rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "gnomad_allele_frequency" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "gnomad_allele_frequency" + assert events[0].allele_id is not None and events[0].variant_id is None # Verify that the job completed successfully session.refresh(sample_link_gnomad_variants_run_pipeline) @@ -557,8 +571,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_independ assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run) @@ -594,8 +608,8 @@ async def test_link_gnomad_variants_with_arq_context_exception_handling_pipeline assert len(session.scalars(select(GnomadAlleleLink)).all()) == 0 # Verify no annotations were rendered - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify that the job errored session.refresh(sample_link_gnomad_variants_run_pipeline) From 6f0f47f50e22e62430d5166d1b59b4c7de7efad7 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:44 -0700 Subject: [PATCH 34/93] feat(vep): derive annotation events from resolution outcomes VEP resolution now returns a VepResolution (consequences vs errored) so a genuine empty is distinguished from an unknown, and linking returns a VepLinkVerdict per allele (CREATED/UNCHANGED). The job derives annotation events from these outcomes instead of re-querying consequence state; errored HGVS are retried, not recorded as negatives. --- src/mavedb/lib/vep.py | 155 ++++++----- .../worker/jobs/external_services/vep.py | 243 +++++++++++------- tests/lib/test_vep.py | 32 ++- .../external_services/network/test_vep.py | 29 ++- .../worker/jobs/external_services/test_vep.py | 143 +++++++---- 5 files changed, 369 insertions(+), 233 deletions(-) diff --git a/src/mavedb/lib/vep.py b/src/mavedb/lib/vep.py index 4b8ac0342..25c0732fa 100644 --- a/src/mavedb/lib/vep.py +++ b/src/mavedb/lib/vep.py @@ -5,9 +5,9 @@ import logging import os from datetime import date -from typing import Mapping, Optional, Sequence +from enum import Enum +from typing import Mapping, NamedTuple, Optional, Sequence -import requests from sqlalchemy import select from sqlalchemy.orm import Session @@ -17,9 +17,9 @@ logger = logging.getLogger(__name__) + ENSEMBL_API_URL = os.environ.get("ENSEMBL_API_URL", "https://rest.ensembl.org") -# List of all possible VEP consequences, in order from most to least severe VEP_CONSEQUENCES = [ "transcript_ablation", "splice_acceptor_variant", @@ -77,6 +77,40 @@ "intron_variant", "intergenic_variant", ] +""" +List of all functional consequences VEP can return, in order of severity (most severe first). +""" + + +class VepLinkVerdict(str, Enum): + """Per-allele outcome of a VEP linking run, returned for every allele whose status is decided. + + The single source of truth for what happened to an allele's consequence this run — the caller + derives annotation status from this, never by re-querying consequence state. An allele absent from + the map had no live consequence and resolved none this run (the caller reads that as "no result"). + + - ``CREATED`` — a new or changed consequence was created/superseded this run. + - ``UNCHANGED`` — a live consequence was retained (value matched, or held against a null run). + """ + + CREATED = "created" + UNCHANGED = "unchanged" + + +class VepResolution(NamedTuple): + """Outcome of resolving a set of HGVS strings, splitting the two kinds of "no consequence". + + This outcome allows us to differentiate between a genuine empty (VEP found nothing) and an + unknown (VEP failed to answer). + + - ``consequences`` — HGVS that resolved to a most-severe consequence (the hits). + - ``errored`` — HGVS whose VEP/Recoder request *failed* (HTTP/transport error after retries); the + result is unknown and the allele should be retried, not treated as a negative. + - Any queried HGVS in neither set was answered (HTTP 200) with no consequence — a genuine **empty**. + """ + + consequences: dict[str, str] + errored: set[str] async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str]]: @@ -86,35 +120,30 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str missing_hgvs (Sequence[str]): List of HGVS strings to recode. Returns: - dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). - Returns an empty dict if Ensembl rejects the batch (e.g. 400 for - unrecognised identifiers) — callers treat missing entries as failures. + dict[str, list[str]]: Mapping of input HGVS to list of genomic HGVS strings (hgvsg). An input + with no recodable genomic mapping is simply absent (a genuine empty). + + Raises: + requests.exceptions.RequestException: if the Recoder request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ headers = {"Content-Type": "application/json", "Accept": "application/json"} + # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/variant_recoder/human", - headers=headers, - json={"ids": list(missing_hgvs)}, - timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier format) means the batch - # cannot be recoded. Return empty so callers can handle these missing entries. - logger.warning( - f"Variant Recoder returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(missing_hgvs)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return {} + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/variant_recoder/human", + headers=headers, + json={"ids": list(missing_hgvs)}, + timeout=600, # Variant Recoder can be very slow for large batches and 504s are common; generous timeout and backoff retries are needed + ), + ) data = response.json() # request_with_backoff handles http errors, so no need to check response status @@ -124,11 +153,13 @@ async def run_variant_recoder(missing_hgvs: Sequence[str]) -> dict[str, list[str hgvs_string = variant_data.get("input") if isinstance(variant_data, dict) else None if variant_str == "input" or not hgvs_string: continue + genomic_strings = variant_data.get("hgvsg") if isinstance(variant_data, dict) else None if genomic_strings: for genomic_hgvs in genomic_strings: if genomic_hgvs.startswith("NC_"): hgvs_to_genomic.setdefault(hgvs_string, []).append(genomic_hgvs) + return hgvs_to_genomic @@ -143,11 +174,14 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O hgvs_strings (Sequence[str]): List of HGVS strings to process (max 200 per call). Returns: - dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. - If no consequence found, maps to None. Returns an empty dict - if Ensembl rejects the batch (e.g. 400 for unrecognised - identifiers) — callers treat missing entries as needing Recoder - fallback or as failures. + dict[str, Optional[str]]: Mapping of HGVS string to functional consequence. An HGVS the + successful response carried no consequence for maps to None (a genuine + miss — the caller may try Recoder, else treats it as empty). + + Raises: + requests.exceptions.RequestException: if the VEP request fails (HTTP/transport error after + retries). The caller attributes the failure to this batch's inputs so they are reported as + errored (unknown, retry) rather than silently conflated with a genuine empty. """ if len(hgvs_strings) > 200: raise ValueError( @@ -160,27 +194,17 @@ async def get_functional_consequence(hgvs_strings: Sequence[str]) -> dict[str, O # request_with_backoff is synchronous (requests lib + time.sleep backoff); run_in_executor # keeps the event loop free during the full request + any retry wait time. loop = asyncio.get_running_loop() - try: - response = await loop.run_in_executor( - None, - functools.partial( - request_with_backoff, - method="POST", - url=f"{ENSEMBL_API_URL}/vep/human/hgvs", - headers=headers, - json={"hgvs_notations": list(hgvs_strings)}, - timeout=60, # VEP can be slow for large batches. - ), - ) - except requests.exceptions.HTTPError as exc: - # A 4xx from Ensembl (e.g. 400 for an unrecognised identifier) means the batch cannot - # be resolved. Return empty so the callers can handle these missing entries. - logger.warning( - f"VEP returned {exc.response.status_code if exc.response is not None else 'unknown'} " - f"for batch of {len(hgvs_strings)} HGVS strings — treating as no results.", - exc_info=exc, - ) - return result + response = await loop.run_in_executor( + None, + functools.partial( + request_with_backoff, + method="POST", + url=f"{ENSEMBL_API_URL}/vep/human/hgvs", + headers=headers, + json={"hgvs_notations": list(hgvs_strings)}, + timeout=60, # VEP can be slow for large batches. + ), + ) data = response.json() for entry in data: @@ -222,7 +246,7 @@ def link_vep_consequences_to_alleles( *, source_version: str, access_date: date, -) -> set[int]: +) -> dict[int, VepLinkVerdict]: """Store VEP consequences against deduplicated alleles, superseding only on change. ``consequence_by_allele_id`` maps each queried allele to the consequence VEP resolved this run @@ -239,12 +263,16 @@ def link_vep_consequences_to_alleles( - **None this run**: leave any live row in place — do not overwrite a held consequence with a null result. Log a warning if VEP found no consequence for an allele which previously had a live consequence. - Does not commit. Returns the allele ids whose consequence was created or superseded this run. + Does not commit. Returns a verdict per allele whose status is decided this run: + :attr:`VepLinkVerdict.CREATED` for a created/superseded consequence, :attr:`~VepLinkVerdict.UNCHANGED` + for a live consequence retained (value matched, or held against a null run). An allele absent from + the map had no live row and resolved nothing — the caller reads that as "no result". This is the + single source of truth for per-allele status; callers must not re-derive it from consequence state. """ save_to_logging_context({"num_alleles_to_link_vep": len(consequence_by_allele_id)}) logger.debug(msg="Linking VEP consequences to alleles", extra=logging_context()) - changed_allele_ids: set[int] = set() + verdicts: dict[int, VepLinkVerdict] = {} for allele_id, consequence in consequence_by_allele_id.items(): live = db.scalar( select(VepAlleleConsequence).where( @@ -253,7 +281,9 @@ def link_vep_consequences_to_alleles( ) ) - # VEP found nothing this run. Do not overwrite a held consequence with a null result. + # TODO#780 - VEP found nothing this run. Do not overwrite a held consequence with a null result; a retained + # consequence is UNCHANGED (status preexisting), while no live row at all leaves the allele out of + # the map (the caller reads that as a no-result). if consequence is None: if live is not None: logger.warning( @@ -261,6 +291,7 @@ def link_vep_consequences_to_alleles( f"'{live.functional_consequence}' in place.", extra=logging_context(), ) + verdicts[allele_id] = VepLinkVerdict.UNCHANGED continue @@ -268,6 +299,7 @@ def link_vep_consequences_to_alleles( if live is not None and live.functional_consequence == consequence: live.source_version = source_version live.access_date = access_date + verdicts[allele_id] = VepLinkVerdict.UNCHANGED continue # New or changed consequence: retire any live row, insert the successor. @@ -283,11 +315,12 @@ def link_vep_consequences_to_alleles( ], VepAlleleConsequence.allele_id == allele_id, ) - changed_allele_ids.add(allele_id) + verdicts[allele_id] = VepLinkVerdict.CREATED - save_to_logging_context({"changed_allele_count": len(changed_allele_ids)}) + changed_allele_count = sum(1 for v in verdicts.values() if v is VepLinkVerdict.CREATED) + save_to_logging_context({"changed_allele_count": changed_allele_count}) logger.info( - msg=f"Created or superseded {len(changed_allele_ids)} VEP allele consequences this run.", + msg=f"Created or superseded {changed_allele_count} VEP allele consequences this run.", extra=logging_context(), ) - return changed_allele_ids + return verdicts diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index 02f6a4498..cf3757335 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -8,8 +8,10 @@ import asyncio import logging import os +from collections import Counter from datetime import date +import requests from sqlalchemy import select from mavedb.lib.annotation_status_manager import AnnotationStatusManager @@ -18,13 +20,16 @@ from mavedb.lib.utils import batched from mavedb.lib.vep import ( VEP_CONSEQUENCES, + VepLinkVerdict, + VepResolution, get_ensembl_release, get_functional_consequence, link_vep_consequences_to_alleles, run_variant_recoder, ) from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.score_set import ScoreSet from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.worker.jobs.utils.setup import validate_job_params @@ -40,36 +45,33 @@ def _annotate_vep( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, + source_version: str, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a VEP_FUNCTIONAL_CONSEQUENCE annotation out to every variant served by an allele. + """Record one VEP_FUNCTIONAL_CONSEQUENCE event for an allele (the consequence is an allele-level fact). - AAS migration seam: the single choke point for VEP's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (which variant's mapping drove the linkage). See - docs/design/allele-annotation-status.md. - - VEP carries no version string, so the VAS row is written with ``version=None`` (unlike gnomAD, - which keys on its data version). + The single choke point for VEP's status writes. One event per allele, stamped with the Ensembl + release queried; provenance (which variants drove the linkage) is derived from the live links + as-of the event, not fanned out here. The consequence value itself is not embedded — it lives in + the ``VepAlleleConsequence`` value table, joinable by ``allele_id`` + ``source_version``. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: @@ -79,22 +81,40 @@ def _vep_hgvs_payload(row: ScoreSetAlleleRow) -> str | None: return row.hgvs_g or row.hgvs_c or row.hgvs_p or None -async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> dict[str, str]: - """Resolve a set of HGVS strings to their most-severe VEP consequence. +async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) -> VepResolution: + """Resolve a set of HGVS strings to their most-severe VEP consequence, splitting empty from errored. Phase 1 submits the HGVS strings to VEP. Phase 2 runs Variant Recoder on the misses (a VEP entry with a null consequence is treated as a miss — VEP knew the variant but could not classify it). Phase 3 re-submits the recoded genomic strings to VEP and maps the most-severe consequence back to - the original HGVS. Returns only the HGVS strings that resolved to a consequence; absent keys are - failures the caller treats as no-result. + the original HGVS. + + Returns a :class:`VepResolution`: ``consequences`` for the hits, and ``errored`` for HGVS whose VEP + or Recoder *request failed* (HTTP/transport error after retries) — those are unknown and must be + retried, never conflated with a genuine empty. An input in neither set was queried successfully and + yielded no consequence (a genuine empty). A failed batch does not abort the run: only that batch's + inputs are marked errored and processing continues. """ all_consequences: dict[str, str] = {} + errored: set[str] = set() batches = list(batched(unique_hgvs, _VEP_BATCH_SIZE)) # --- Phase 1: initial VEP pass --- all_missing_hgvs: set[str] = set() for batch_idx, batch in enumerate(batches): - consequences = await get_functional_consequence(list(batch)) + try: + consequences = await get_functional_consequence(list(batch)) + except requests.exceptions.RequestException as exc: + # The request failed — we do not know these alleles' consequence. Mark errored and do + # not route to Recoder or count as a genuine empty. + logger.warning( + msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + hit_consequences = {h: c for h, c in consequences.items() if c is not None} all_consequences.update(hit_consequences) all_missing_hgvs.update(set(batch) - set(hit_consequences.keys())) @@ -111,7 +131,7 @@ async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) ) if not all_missing_hgvs: - return all_consequences + return VepResolution(all_consequences, errored) # --- Phase 2: Variant Recoder fallback for HGVS strings VEP could not resolve --- recoder_batch_list = list(batched(list(all_missing_hgvs), _RECODER_BATCH_SIZE)) @@ -136,18 +156,19 @@ async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, lis return_exceptions=True, ) - first_exception = next((r for r in recoder_results if isinstance(r, BaseException)), None) - if first_exception is not None: - successful_batches = sum(1 for r in recoder_results if not isinstance(r, BaseException)) - logger.error( - msg=f"Variant Recoder error ({successful_batches}/{total_recoder_batches} batches succeeded): {first_exception}", - extra=job_manager.logging_context(), - ) - raise first_exception - + # A failed Recoder batch marks its inputs errored (they were misses; we still don't know them) and + # the run continues — one bad batch must not abort every allele's annotation. hgvs_to_genomic: dict[str, list[str]] = {} - for result in recoder_results: - hgvs_to_genomic.update(result) # type: ignore[arg-type] + for batch, result in zip(recoder_batch_list, recoder_results): + if isinstance(result, BaseException): + logger.warning( + msg=f"Variant Recoder request failed for a batch of {len(batch)} HGVS; marking errored.", + exc_info=result, + extra=job_manager.logging_context(), + ) + errored.update(batch) + continue + hgvs_to_genomic.update(result) logger.info( msg=f"Completed Variant Recoder processing. {len(hgvs_to_genomic)} HGVS strings successfully recoded.", @@ -158,23 +179,37 @@ async def _recoder_with_semaphore(batch: list[str], total: int) -> dict[str, lis all_recoded_genomic_hgvs = list({g for genomic_list in hgvs_to_genomic.values() for g in genomic_list}) recoded_vep_batch_list = list(batched(all_recoded_genomic_hgvs, _VEP_BATCH_SIZE)) all_recoded_consequences: dict[str, str | None] = {} + errored_genomic: set[str] = set() for recoded_idx, recoded_batch in enumerate(recoded_vep_batch_list): - all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + try: + all_recoded_consequences.update(await get_functional_consequence(list(recoded_batch))) + except requests.exceptions.RequestException as exc: + logger.warning( + msg=f"VEP request failed for recoded batch {recoded_idx + 1}/{len(recoded_vep_batch_list)} " + f"({len(recoded_batch)} HGVS); marking errored.", + exc_info=exc, + extra=job_manager.logging_context(), + ) + errored_genomic.update(recoded_batch) job_manager.update_progress( 66 + int((recoded_idx + 1) / len(recoded_vep_batch_list) * 33), 100, f"Processed recoded VEP batch {recoded_idx + 1}/{len(recoded_vep_batch_list)}", ) - # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. + # Map the most-severe consequence from the recoded genomic strings back to the original HGVS. An + # original HGVS is a hit if any recoded form resolved; else errored if a recoded query failed and + # nothing resolved; else a genuine empty (Recoder ran, VEP found no consequence) — left out of both. for original_hgvs, recoded_hgvs_list in hgvs_to_genomic.items(): recoded_consequences = [c for h in recoded_hgvs_list if (c := all_recoded_consequences.get(h))] most_severe = next((c for c in VEP_CONSEQUENCES if c in recoded_consequences), None) if most_severe: all_consequences[original_hgvs] = most_severe + elif any(g in errored_genomic for g in recoded_hgvs_list): + errored.add(original_hgvs) - return all_consequences + return VepResolution(all_consequences, errored) @with_pipeline_management @@ -222,14 +257,22 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan job_manager.update_progress(0, 100, "Starting VEP consequence linkage.") logger.info(msg="Started VEP consequence linkage", extra=job_manager.logging_context()) - # One work-unit per allele (payload = HGVS; alleles without one are skipped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele VEP is most reliable on - # is often the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # One work-unit per allele (payload = HGVS; alleles without one are skipped). Events are allele-keyed, + # so each allele records its own event. allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=_vep_hgvs_payload, ) + annotation_counts: Counter[str] = Counter( + { + "created_allele_count": 0, + "preexisting_allele_count": 0, + "absent_allele_count": 0, + "errored_allele_count": 0, + } + ) + num_alleles_with_hgvs = len(allele_data) job_manager.save_to_context({"num_alleles_to_link_vep": num_alleles_with_hgvs}) @@ -239,9 +282,7 @@ async def populate_vep_for_score_set(ctx: dict, job_id: int, job_manager: JobMan extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded( - data={"created_allele_count": 0, "preexisting_allele_count": 0, "skipped_allele_count": 0} - ) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) all_allele_ids = set(allele_data.keys()) @@ -265,12 +306,12 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: ).all() ) - # Cost: skip alleles already resolved at the current Ensembl release (they cannot change without a + # Skip alleles already resolved at the current Ensembl release (they cannot change without a # release bump). force re-queries all — including alleles unchanged upstream but whose VEP_CONSEQUENCES # severity ordering we have since edited; the linker still supersedes only on a value change, so a # forced no-op writes nothing. already_current = set() if force else alleles_at_current_release(all_allele_ids) - hgvs_by_allele = {aid: allele_data[aid].payload for aid in allele_data if aid not in already_current} + hgvs_by_allele = {aid: allele_data[aid] for aid in allele_data if aid not in already_current} unique_hgvs = sorted(set(hgvs_by_allele.values())) job_manager.save_to_context( { @@ -280,12 +321,15 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: } ) - changed_allele_ids: set[int] = set() + verdicts: dict[int, VepLinkVerdict] = {} + errored_allele_ids: set[int] = set() if unique_hgvs: job_manager.update_progress(10, 100, f"Querying VEP for {len(unique_hgvs)} HGVS strings.") - consequences_by_hgvs = await _resolve_consequences(unique_hgvs, job_manager) - consequence_by_allele_id = {aid: consequences_by_hgvs.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} - changed_allele_ids = link_vep_consequences_to_alleles( + resolution = await _resolve_consequences(unique_hgvs, job_manager) + consequence_by_allele_id = {aid: resolution.consequences.get(hgvs) for aid, hgvs in hgvs_by_allele.items()} + # Alleles whose VEP/Recoder request failed: unknown, not a negative — kept distinct from empties. + errored_allele_ids = {aid for aid, hgvs in hgvs_by_allele.items() if hgvs in resolution.errored} + verdicts = link_vep_consequences_to_alleles( job_manager.db, consequence_by_allele_id, source_version=ensembl_release, access_date=date.today() ) job_manager.db.flush() @@ -296,58 +340,69 @@ def alleles_at_current_release(allele_ids: set[int]) -> set[int]: ) job_manager.update_progress(99, 100, "All alleles already current at this Ensembl release.") - # Status is an audit event, written every run: SUCCESS/created if linked this run, SUCCESS/preexisting - # if already current or re-confirmed unchanged, SKIPPED if no live consequence (VEP had no result). - live_now = job_manager.db.execute( - select(VepAlleleConsequence.allele_id, VepAlleleConsequence.functional_consequence) - .where(VepAlleleConsequence.allele_id.in_(all_allele_ids)) - .where(VepAlleleConsequence.current) - .where(VepAlleleConsequence.functional_consequence.isnot(None)) - ).all() - consequence_now = {r.allele_id: r.functional_consequence for r in live_now} - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - created_count = preexisting_count = skipped_count = 0 - for allele_id, entry in allele_data.items(): - if allele_id in consequence_now: - action = "created" if allele_id in changed_allele_ids else "preexisting" - if action == "created": - created_count += 1 - else: - preexisting_count += 1 + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, hgvs in allele_data.items(): + verdict = verdicts.get(allele_id) + if verdict is VepLinkVerdict.CREATED: + annotation_counts["created_allele_count"] += 1 _annotate_vep( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={ - "functional_consequence": consequence_now[allele_id], - "hgvs": entry.payload, - "action": action, - }, + allele_id, + Disposition.PRESENT, + EventReason.CREATED, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, ) + + elif allele_id in already_current or verdict is VepLinkVerdict.UNCHANGED: + annotation_counts["preexisting_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=ensembl_release, + metadata={"hgvs": hgvs}, + ) + + elif allele_id in errored_allele_ids: + annotation_counts["errored_allele_count"] += 1 + _annotate_vep( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=ensembl_release, + error_message="The VEP/Variant Recoder request for this allele failed; result unknown.", + metadata={"hgvs": hgvs}, + ) + else: - skipped_count += 1 + annotation_counts["absent_allele_count"] += 1 _annotate_vep( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, - error_message="VEP could not determine a functional consequence for this allele, even after Variant Recoder fallback.", - metadata={"hgvs": entry.payload}, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=ensembl_release, + error_message="VEP found no functional consequence for this allele, even after Variant Recoder fallback.", + metadata={"hgvs": hgvs}, ) annotation_manager.flush() - outcome_data = { - "created_allele_count": created_count, - "preexisting_allele_count": preexisting_count, - "skipped_allele_count": skipped_count, - } + outcome_data = dict(annotation_counts) job_manager.save_to_context(outcome_data) job_manager.update_progress( 100, 100, - f"Completed VEP linkage: {created_count} created, {preexisting_count} preexisting, {skipped_count} skipped.", + ( + f"Completed VEP linkage: {annotation_counts['created_allele_count'] + annotation_counts['preexisting_allele_count']} linked, " + f"{annotation_counts['absent_allele_count']} absent (no result), " + f"{annotation_counts['errored_allele_count']} errored." + ), ) logger.info(msg="Done linking VEP consequences to alleles.", extra=job_manager.logging_context()) job_manager.db.flush() diff --git a/tests/lib/test_vep.py b/tests/lib/test_vep.py index 0d9473f1b..bea141ab2 100644 --- a/tests/lib/test_vep.py +++ b/tests/lib/test_vep.py @@ -11,6 +11,7 @@ from sqlalchemy import select from mavedb.lib.vep import ( + VepLinkVerdict, get_ensembl_release, get_functional_consequence, link_vep_consequences_to_alleles, @@ -290,12 +291,12 @@ def test_link_vep_creates_new_consequence(session): """A consequence for an allele with no live row creates a single live row and is reported changed.""" allele = _make_allele(session, vrs_digest="vrs-1") - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == {allele.id} + assert verdicts == {allele.id: VepLinkVerdict.CREATED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -305,7 +306,8 @@ def test_link_vep_creates_new_consequence(session): def test_link_vep_unchanged_bumps_version_and_date_in_place(session): """Re-confirming an unchanged consequence at a new release advances source_version and access_date - in place — no supersede, no new valid-time boundary, and the allele is not reported changed.""" + in place — no supersede, no new valid-time boundary. The allele is reported UNCHANGED (status + preexisting) so the caller need not re-query consequence state.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -317,12 +319,12 @@ def test_link_vep_unchanged_bumps_version_and_date_in_place(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} # One row, still live, never retired — version and access_date advanced in place. all_rows = _all_rows_for(session, allele.id) assert len(all_rows) == 1 @@ -345,12 +347,12 @@ def test_link_vep_changed_consequence_supersedes(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: "missense_variant"}, source_version="116", access_date=date.today() ) session.commit() - assert changed == {allele.id} + assert verdicts == {allele.id: VepLinkVerdict.CREATED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -363,7 +365,8 @@ def test_link_vep_changed_consequence_supersedes(session): def test_link_vep_none_leaves_live_row_untouched(session): """A transient None result must not overwrite a held consequence: the live row is left intact - (value, version, and date) and the allele is not reported changed.""" + (value, version, and date). The held consequence is reported UNCHANGED (status preexisting) — the + allele still has a live consequence, it just was not re-confirmed this run.""" allele = _make_allele(session, vrs_digest="vrs-1") session.add( VepAlleleConsequence( @@ -375,12 +378,12 @@ def test_link_vep_none_leaves_live_row_untouched(session): ) session.commit() - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: None}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {allele.id: VepLinkVerdict.UNCHANGED} live = _live_rows_for(session, allele.id) assert len(live) == 1 assert live[0].functional_consequence == "missense_variant" @@ -390,14 +393,15 @@ def test_link_vep_none_leaves_live_row_untouched(session): def test_link_vep_none_with_no_live_row_writes_nothing(session): - """A None result for an allele with no live row writes nothing (the allele is re-queried next run), - mirroring gnomAD's no-match handling.""" + """A None result for an allele with no live row writes nothing and leaves the allele out of the + verdict map (the caller reads that as a no-result and re-queries next run), mirroring gnomAD's + no-match handling.""" allele = _make_allele(session, vrs_digest="vrs-1") - changed = link_vep_consequences_to_alleles( + verdicts = link_vep_consequences_to_alleles( session, {allele.id: None}, source_version="116", access_date=date.today() ) session.commit() - assert changed == set() + assert verdicts == {} assert len(_all_rows_for(session, allele.id)) == 0 diff --git a/tests/worker/jobs/external_services/network/test_vep.py b/tests/worker/jobs/external_services/network/test_vep.py index 0a52012af..61ba8dfd4 100644 --- a/tests/worker/jobs/external_services/network/test_vep.py +++ b/tests/worker/jobs/external_services/network/test_vep.py @@ -7,8 +7,9 @@ from sqlalchemy import select from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.vep_allele_consequence import VepAlleleConsequence pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -52,14 +53,14 @@ async def test_populate_vep_e2e( assert live.functional_consequence is not None assert live.access_date is not None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant.id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT async def test_populate_vep_e2e_with_recoder_path( self, @@ -97,11 +98,11 @@ async def test_populate_vep_e2e_with_recoder_path( assert live.functional_consequence is not None assert live.access_date is not None - annotation = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.variant_id == variant.id, - VariantAnnotationStatus.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, - VariantAnnotationStatus.current.is_(True), + # Events are allele-keyed now; the variant resolves its status through the live link. + event = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.allele_id == allele.id, + AnnotationEvent.annotation_type == AnnotationType.VEP_FUNCTIONAL_CONSEQUENCE, ) ).one() - assert annotation.status == AnnotationStatus.SUCCESS + assert event.disposition == Disposition.PRESENT diff --git a/tests/worker/jobs/external_services/test_vep.py b/tests/worker/jobs/external_services/test_vep.py index 7e33f24fa..19587f5c2 100644 --- a/tests/worker/jobs/external_services/test_vep.py +++ b/tests/worker/jobs/external_services/test_vep.py @@ -11,8 +11,9 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.lib.vep import VepResolution from mavedb.worker.jobs.external_services.vep import populate_vep_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager @@ -64,7 +65,8 @@ async def test_no_alleles_with_hgvs( assert result.status == JobStatus.SUCCEEDED assert result.data["created_allele_count"] == 0 assert result.data["preexisting_allele_count"] == 0 - assert result.data["skipped_allele_count"] == 0 + assert result.data["absent_allele_count"] == 0 + assert result.data["errored_allele_count"] == 0 async def test_calls_resolver_when_alleles_present( self, @@ -76,7 +78,7 @@ async def test_calls_resolver_when_alleles_present( setup_sample_alleles_for_vep, ): """The VEP resolver is invoked once when the score set has HGVS-bearing alleles to query.""" - with patch(_RESOLVE, return_value={}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({}, set())) as mock_resolve: result = await populate_vep_for_score_set( mock_worker_ctx, 1, @@ -152,7 +154,7 @@ async def test_no_alleles_with_hgvs( assert result.status == JobStatus.SUCCEEDED assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.SUCCEEDED @@ -166,19 +168,52 @@ async def test_no_consequence_resolved( sample_populate_vep_run, setup_sample_alleles_for_vep, ): - """An allele with HGVS that VEP cannot classify gets no consequence row and a SKIPPED VAS.""" + """A genuine empty (VEP queried successfully, found nothing) writes no consequence row and an + ABSENT/no_record event — a trustworthy negative, now that empty is split from a failed request.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={}): + with patch(_RESOLVE, return_value=VepResolution({}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED + assert result.data["absent_allele_count"] == 1 assert len(_live_consequences_for(session, allele.id)) == 0 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "skipped" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "absent" + assert events[0].reason == "no_record" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None + + async def test_request_failure_recorded_as_errored( + self, + session, + with_populated_domain_data, + with_populate_vep_job, + mock_worker_ctx, + sample_populate_vep_run, + setup_sample_alleles_for_vep, + ): + """An allele whose VEP/Recoder request *failed* is FAILED/api_error — distinct from a genuine + empty — so a transient outage is never mistaken for a confirmed absence.""" + _, allele = setup_sample_alleles_for_vep + + # Resolver reports the allele's HGVS as errored (request failed), not as a hit or an empty. + with patch(_RESOLVE, return_value=VepResolution({}, {allele.hgvs_c})): + result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) + + assert result.status == JobStatus.SUCCEEDED + assert result.data["errored_allele_count"] == 1 + assert result.data["absent_allele_count"] == 0 + assert len(_live_consequences_for(session, allele.id)) == 0 + + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "failed" + assert events[0].reason == "api_error" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None async def test_successful_linking_independent( self, @@ -192,7 +227,7 @@ async def test_successful_linking_independent( """A resolved consequence creates a single live VepAlleleConsequence and a SUCCESS VAS row.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -204,13 +239,14 @@ async def test_successful_linking_independent( assert live[0].source_version == _ENSEMBL_RELEASE assert live[0].access_date == date.today() - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" - assert annotation_statuses[0].annotation_metadata["action"] == "created" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id == allele.id and events[0].variant_id is None - async def test_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_links_and_annotates_rt_derived_allele( self, session, with_populated_domain_data, @@ -219,15 +255,16 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( sample_populate_vep_run, setup_rt_derived_allele_for_vep, ): - """VEP linkage must cover the full allele set, not just authoritative links: the RT-derived - allele's genomic HGVS resolves and must get a consequence row. Per-variant VAS, however, is - written only for the authoritative link (the interim bandaid) — so exactly one annotation row - is produced, keyed to the variant, never a second row for the RT-derived allele.""" + """VEP linkage covers the full allele set, not just authoritative links: the RT-derived allele's + genomic HGVS resolves and gets a consequence row. Events are allele-keyed, so each allele in the + score set gets its own event — present for the resolved RT allele, absent/no_record for the + authoritative allele VEP queried but found nothing (a genuine empty, distinct from a request + failure). The per-variant bandaid's limitation (dropping the RT-derived allele's status) is lifted.""" variant, authoritative_allele, rt_allele = setup_rt_derived_allele_for_vep # VEP resolves only the RT-derived allele's genomic HGVS; the authoritative allele's coding - # HGVS yields nothing this run. - with patch(_RESOLVE, return_value={rt_allele.hgvs_g: "missense_variant"}): + # HGVS is queried successfully but yields no consequence this run (a genuine empty, not errored). + with patch(_RESOLVE, return_value=VepResolution({rt_allele.hgvs_g: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.status == JobStatus.SUCCEEDED @@ -239,13 +276,15 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( # The authoritative allele's HGVS had no consequence, so it gets no row. assert len(_live_consequences_for(session, authoritative_allele.id)) == 0 - # Annotation status is written only for the authoritative link: exactly one VAS row, for the - # variant. (Its status is "skipped" because the variant's authoritative allele had no - # consequence — cross-level resolution onto the RT-derived allele is deferred to AnnotationEvent.) - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].variant_id == variant.id - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + # One allele-keyed event per allele (never per-variant): the resolved RT allele is present, the + # unclassifiable authoritative allele is absent/no_record (queried, genuinely empty). + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == "vep_functional_consequence" for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == "present" + assert by_allele[authoritative_allele.id].disposition == "absent" + assert by_allele[authoritative_allele.id].reason == "no_record" async def test_skips_allele_already_at_current_release( self, @@ -283,10 +322,10 @@ async def test_skips_allele_already_at_current_release( assert len(rows) == 1 assert rows[0].valid_to is None - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_metadata["action"] == "preexisting" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "preexisting" async def test_new_release_same_consequence_bumps_in_place( self, @@ -312,7 +351,7 @@ async def test_new_release_same_consequence_bumps_in_place( ) session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # older release -> not skipped, re-queried @@ -354,7 +393,7 @@ async def test_force_requeries_unchanged_without_churn( sample_populate_vep_run.job_params = {**sample_populate_vep_run.job_params, "force": True} session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}) as mock_resolve: + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())) as mock_resolve: result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) mock_resolve.assert_called_once() # force bypassed the current-release skip @@ -391,7 +430,7 @@ async def test_new_release_changed_consequence_supersedes( ) session.commit() - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run.id) assert result.data["created_allele_count"] == 1 @@ -420,16 +459,18 @@ async def test_successful_linking_pipeline( """End-to-end successful linking within a pipeline updates both job and pipeline status.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): result = await populate_vep_for_score_set(mock_worker_ctx, sample_populate_vep_run_pipeline.id) assert result.status == JobStatus.SUCCEEDED assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.SUCCEEDED @@ -479,17 +520,19 @@ async def test_populate_vep_with_arq_context_independent( """The VEP job links a consequence and records a SUCCESS annotation through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run.id) await arq_worker.async_run() await arq_worker.run_check() assert len(session.scalars(select(VepAlleleConsequence).where(VepAlleleConsequence.current)).all()) == 1 - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == 1 - assert annotation_statuses[0].status == "success" - assert annotation_statuses[0].annotation_type == "vep_functional_consequence" + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == "present" + assert events[0].reason == "created" + assert events[0].annotation_type == "vep_functional_consequence" + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.SUCCEEDED @@ -507,7 +550,7 @@ async def test_populate_vep_with_arq_context_pipeline( """The VEP job completes and advances the pipeline through the ARQ worker.""" _, allele = setup_sample_alleles_for_vep - with patch(_RESOLVE, return_value={allele.hgvs_c: "missense_variant"}): + with patch(_RESOLVE, return_value=VepResolution({allele.hgvs_c: "missense_variant"}, set())): await arq_redis.enqueue_job("populate_vep_for_score_set", sample_populate_vep_run_pipeline.id) await arq_worker.async_run() await arq_worker.run_check() @@ -540,7 +583,7 @@ async def test_populate_vep_with_arq_context_exception_handling_independent( mock_send_slack_job_error.assert_called_once() assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run) assert sample_populate_vep_run.status == JobStatus.ERRORED @@ -566,7 +609,7 @@ async def test_populate_vep_with_arq_context_exception_handling_pipeline( mock_send_slack_job_error.assert_called_once() assert len(session.scalars(select(VepAlleleConsequence)).all()) == 0 - assert len(session.query(VariantAnnotationStatus).all()) == 0 + assert len(session.scalars(select(AnnotationEvent)).all()) == 0 session.refresh(sample_populate_vep_run_pipeline) assert sample_populate_vep_run_pipeline.status == JobStatus.ERRORED From ed50102b7f13ba2cff4dfab57eb3d27b171ead5a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:30:55 -0700 Subject: [PATCH 35/93] feat(clingen): emit annotation events from CAR/LDH jobs The ClinGen allele-registration and LDH-submission jobs now record AnnotationEvents via record_event, using the shared EventReason vocabulary (created/preexisting/ reconfirmed/submitted, and the CAR-specific failure and reconfirmation-skipped reasons) in place of per-variant status updates. --- .../worker/jobs/external_services/clingen.py | 220 ++++++------ .../jobs/external_services/test_clingen.py | 316 +++++++++--------- 2 files changed, 268 insertions(+), 268 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 42cdcb36a..768d13278 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -11,7 +11,8 @@ import asyncio import functools import logging -from dataclasses import dataclass, field +from collections.abc import Sequence +from dataclasses import dataclass from sqlalchemy import select @@ -32,7 +33,9 @@ from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -48,44 +51,33 @@ class _AlleleEntry: post_mapped: dict | None existing_caid: str | None - # Variants for which THIS allele is the authoritative measurement — the only ones that receive a - # per-variant VAS row. INTERIM BANDAID (do not deploy as final): keying clingen's per-variant - # status to the single authoritative link sidesteps the multiple "current" rows a full allele - # fan-out would write for one variant. Durable fix is an allele-level event log; rationale and - # migration seam in docs/design/allele-annotation-status.md. - authoritative_variant_ids: list[int] = field(default_factory=list) def _annotate_caid( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a CLINGEN_ALLELE_ID annotation out to every variant served by an allele. + """Record one CLINGEN_ALLELE_ID event for an allele (the CAID is an allele-level fact). - AAS migration seam: the single choke point for clingen's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance (who caused the registration). See - docs/design/allele-annotation-status.md. + The single choke point for CAR's status writes. Provenance (which variants drove the + registration) is derived from the live links as-of the event, not fanned out here. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.CLINGEN_ALLELE_ID, - version=None, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele_id, + disposition=disposition, + reason=reason, + metadata=meta or None, + ) @with_pipeline_management @@ -172,53 +164,59 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: if row.allele_id not in allele_data: allele_data[row.allele_id] = _AlleleEntry(post_mapped=row.post_mapped, existing_caid=row.clingen_allele_id) - if row.is_authoritative: - allele_data[row.allele_id].authoritative_variant_ids.append(row.variant_id) - - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) # Track outcomes by distinct allele_id. clingen_allele_id is an allele-level fact (the CAID # lives on the Allele) and CAR's operation is per-allele, so the reported counts are in allele - # units — and they cover every allele submitted, including the RT-derived ones that produce no - # per-variant status row. Each allele has exactly one outcome (submitted once → one response), so - # these sets are disjoint by construction. (Per-variant VAS rows are still written via the - # authoritative link below — that is the interim bandaid, separate from these operation counts.) + # units — and they cover every allele submitted, including the RT-derived ones. Each allele has + # exactly one outcome (submitted once → one response), so these sets are disjoint by construction. linked_allele_ids: set[int] = set() preexisting_allele_ids: set[int] = set() failed_allele_ids: set[int] = set() + preexisting_alleles = [] + pending_alleles = [] + for aid, entry in allele_data.items(): + if entry.existing_caid: + preexisting_alleles.append(aid) + elif force_reregister: + pending_alleles.append(aid) + elif not entry.existing_caid: + pending_alleles.append(aid) + # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. - preexisting = [aid for aid, entry in allele_data.items() if entry.existing_caid] if not force_reregister else [] + preexisting = preexisting_alleles if not force_reregister else [] for allele_id in preexisting: entry = allele_data[allele_id] preexisting_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "preexisting"}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + metadata={"clingen_allele_id": entry.existing_caid}, ) - # Alleles that need CAR submission: new ones, or all when force_reregister=True. - pending_allele_ids = [aid for aid, entry in allele_data.items() if force_reregister or not entry.existing_caid] - job_manager.update_progress(10, 100, f"Preparing {len(pending_allele_ids)} alleles for CAR submission.") + job_manager.update_progress(10, 100, f"Preparing {len(pending_alleles)} alleles for CAR submission.") # Build HGVS → [allele_ids] map. Multi-variant cis-phased blocks produce no HGVS # (combine_cis defaults to False); those alleles are annotated as failures immediately. hgvs_to_allele_ids: dict[str, list[int]] = {} - for allele_id in pending_allele_ids: + for allele_id in pending_alleles: entry = allele_data[allele_id] hgvs = get_hgvs_from_post_mapped(entry.post_mapped) if hgvs: hgvs_to_allele_ids.setdefault(hgvs, []).append(allele_id) - # Allele is registered but post_mapped can no longer produce HGVS — data - # regression worth surfacing, but the CAID is still valid so treat it as - # preexisting rather than failing the variant. + # TODO#780 - Allele is registered but post_mapped can no longer produce HGVS — data + # regression worth surfacing, but the CAID is still valid so treat it as preexisting + # rather than failing the variant. elif entry.existing_caid: - preexisting_allele_ids.add(allele_id) + preexisting_alleles.append(allele_id) logger.warning( msg=( f"Could not construct HGVS for allele {allele_id} during force re-registration " @@ -228,12 +226,14 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: ) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": entry.existing_caid, "registration_source": "reconfirmation_skipped"}, + allele_id, + Disposition.PRESENT, + EventReason.RECONFIRMATION_SKIPPED, + metadata={"clingen_allele_id": entry.existing_caid}, ) - # No HGVS-- un-submittable. + # No HGVS-- un-submittable. The allele cannot produce an identifier to register, + # so treat this as not_applicable rather than a failure. else: failed_allele_ids.add(allele_id) logger.warning( @@ -242,9 +242,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: ) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.MISSING_IDENTIFIER, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.NO_HGVS, error_message="Could not extract a valid HGVS string from post-mapped allele data.", ) @@ -315,9 +315,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, error_message="Failed to register allele with ClinGen Allele Registry.", metadata={ "submitted_hgvs": hgvs_string, @@ -339,9 +339,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", metadata={"submitted_hgvs": hgvs_string}, ) @@ -353,7 +353,7 @@ def _outcome_data() -> dict[str, int]: entry = allele_data[allele_id] prior_caid = entry.existing_caid - # CAID is immutable — a different value returned by CAR is a hard invariant + # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant # violation. Do not overwrite; record a failure with full audit context. if prior_caid and prior_caid != caid: logger.error( @@ -368,9 +368,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED, + allele_id, + Disposition.FAILED, + EventReason.CAID_CONFLICT, error_message="CAR returned a CAID that conflicts with the stored value.", metadata={ "clingen_allele_id": prior_caid, @@ -385,7 +385,7 @@ def _outcome_data() -> dict[str, int]: allele = alleles_by_id[allele_id] allele.clingen_allele_id = caid - registration_source = "reconfirmed" if prior_caid else "this_run" + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED if prior_caid: logger.info( msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", @@ -394,9 +394,10 @@ def _outcome_data() -> dict[str, int]: _annotate_caid( annotation_manager, - entry.authoritative_variant_ids, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "registration_source": registration_source}, + allele_id, + Disposition.PRESENT, + reason, + metadata={"clingen_allele_id": caid}, ) # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up @@ -408,9 +409,9 @@ def _outcome_data() -> dict[str, int]: failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, - allele_data[allele_id].authoritative_variant_ids, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, error_message="Failed to register allele with ClinGen Allele Registry.", metadata={"submitted_hgvs": hgvs_string}, ) @@ -496,17 +497,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MappingRecord. RT-derived equivalence alleles are intentionally excluded — LDH links each # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). - variant_objects = job_manager.db.execute( - select(Variant, MappingRecord, AlleleModel) - .join(MappingRecord, MappingRecord.variant_id == Variant.id) - .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) - .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) - .where(Variant.score_set_id == score_set.id) - .where(MappingRecord.current) - .where(MappingRecordAllele.current) - .where(MappingRecordAllele.is_authoritative.is_(True)) - .where(AlleleModel.post_mapped.is_not(None)) - ).all() + variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( + job_manager.db.execute( + select(Variant, MappingRecord, AlleleModel) + .join(MappingRecord, MappingRecord.variant_id == Variant.id) + .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(AlleleModel, AlleleModel.id == MappingRecordAllele.allele_id) + .where(Variant.score_set_id == score_set.id) + .where(MappingRecord.current) + .where(MappingRecordAllele.current) + .where(MappingRecordAllele.is_authoritative.is_(True)) + .where(AlleleModel.post_mapped.is_not(None)) + ) + .tuples() + .all() + ) # Track total variants to submit job_manager.save_to_context({"total_variants_to_submit_ldh": len(variant_objects)}) @@ -521,11 +526,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: job_manager.update_progress(10, 100, f"Submitting {len(variant_objects)} mapped variants to LDH.") # Build submission content - variant_content = [] - variant_for_urn = {} + variant_content: list[tuple[str, Variant, MappingRecord, AlleleModel]] = [] + variant_for_urn: dict[str, Variant] = {} for variant, mapping_record, allele in variant_objects: - # See the note above: cis-phased blocks are skipped here pending ClinGen guidance - # (https://github.com/VariantEffect/mavedb-api/issues/764). + # cis-phased blocks are skipped here pending ClinGen guidance TODO#764 variation = get_hgvs_from_post_mapped(allele.post_mapped) if not variation: @@ -536,7 +540,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: continue variant_content.append((variation, variant, mapping_record, allele)) - variant_for_urn[variant.urn] = variant + # TODO#372: nullable URNs + variant_for_urn[variant.urn] = variant # type: ignore if not variant_content: logger.warning( @@ -561,8 +566,10 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: "ldh_submission_failures": len(submission_failures), }) - # TODO prior to finalizing: Verify typing of ClinGen submission responses. See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) submitted_variant_urns = set() for success in submission_successes: logger.debug( @@ -572,24 +579,21 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: submitted_urn = success["data"]["entId"] submitted_variant = variant_for_urn.get(submitted_urn) + # LDH echoed back an entId we never submitted — record it for investigation rather + # than crashing the whole job mid-batch. if submitted_variant is None: - # LDH echoed back an entId we never submitted — record it for investigation rather - # than crashing the whole job mid-batch. logger.warning( msg=f"LDH returned an unrecognized entId not in this submission: {submitted_urn!r}.", extra=job_manager.logging_context(), ) continue - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=submitted_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.SUCCESS, - annotation_data={ - "annotation_metadata": {"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, - }, - current=True, + disposition=Disposition.PRESENT, + reason=EventReason.SUBMITTED, + metadata={"ldh_iri": success["data"]["ldhIri"], "ldh_id": success["data"]["ldhId"]}, ) submitted_variant_urns.add(submitted_urn) @@ -606,16 +610,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: failed_variant = variant_for_urn[failure_urn] - annotation_manager.add_annotation( + annotation_manager.record_event( + AnnotationType.LDH_SUBMISSION, variant_id=failed_variant.id, - annotation_type=AnnotationType.LDH_SUBMISSION, - version=None, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, - annotation_data={ - "error_message": "Failed to submit variant to ClinGen Linked Data Hub.", - }, - current=True, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "Failed to submit variant to ClinGen Linked Data Hub."}, ) annotation_manager.flush() diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 76b1b0b2d..3b44a35c5 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -16,7 +16,7 @@ from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clingen import ( submit_score_set_mappings_to_car, submit_score_set_mappings_to_ldh, @@ -153,14 +153,14 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() assert len(alleles) == 0 - # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 4 variants dedup to 1 allele → one allele-keyed event, failed (no CAR response). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_all_car_errors( self, @@ -221,16 +221,16 @@ async def test_submit_score_set_mappings_to_car_all_car_errors( alleles = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() assert len(alleles) == 0 - # 1 allele failed → all 4 variant annotations are failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele, rejected by CAR → one failed event (reason=service_rejected). + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "service_rejected" - async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_annotation( + async def test_submit_score_set_mappings_to_car_event_per_allele( self, mock_worker_ctx, session, @@ -243,8 +243,8 @@ async def test_submit_score_set_mappings_to_car_derived_allele_no_duplicate_anno dummy_variant_creation_job_run, dummy_variant_mapping_job_run, ): - """A variant linked to an authoritative AND a derived allele gets exactly one VAS row (from the - authoritative link), while the derived allele is still registered with a CAID.""" + """Each allele — authoritative and derived — gets exactly one allele-keyed event; there is no + per-variant fan-out. Both alleles are registered with a CAID.""" await create_mappings_in_score_set( session, mock_s3_client, @@ -300,18 +300,16 @@ def fake_dispatch(hgvs_list): assert result.status == JobStatus.SUCCEEDED - # Exactly one current VAS row per variant (4 total) — no duplicate from the derived allele. - current_statuses = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.current.is_(True), - ) + # One event per allele (2 alleles → 2 events), each keyed on its allele, not fanned per-variant. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(current_statuses) == 4 - assert len({s.variant_id for s in current_statuses}) == 4 + assert len(events) == 2 + assert all(e.variant_id is None for e in events) + assert {e.allele_id for e in events} == {authoritative_allele.id, derived_allele.id} + assert all(e.disposition == "present" for e in events) - # Both alleles registered: registration breadth is preserved even though the derived allele - # produced no VAS row. + # Both alleles registered: registration breadth covers the derived allele too. session.refresh(authoritative_allele) session.refresh(derived_allele) assert authoritative_allele.clingen_allele_id is not None @@ -370,13 +368,13 @@ async def test_submit_score_set_mappings_to_car_response_count_mismatch( # No CAID written — neither of the returned values is trusted. assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.failure_category == "external_api_error" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "api_error" async def test_submit_score_set_mappings_to_car_malformed_response( self, @@ -425,15 +423,15 @@ async def test_submit_score_set_mappings_to_car_malformed_response( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.FAILED - # No CAID assigned, and all 4 variant annotations are failed as rejected. + # No CAID assigned; the one allele's event is failed (reason=malformed_response). assert len(session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all()) == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.failure_category == "external_service_rejected" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "malformed_response" async def test_submit_score_set_mappings_to_car_repeated_hgvs( self, @@ -492,14 +490,14 @@ async def test_submit_score_set_mappings_to_car_repeated_hgvs( assert len(alleles) == 1 assert alleles[0].clingen_allele_id == "CA_DUPLICATE" - # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + # 1 allele → one present event. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_partial_failure( self, @@ -562,13 +560,13 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # All 4 variant annotations succeeded - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 4 + assert len(present_events) == 1 async def test_submit_score_set_mappings_to_car_hgvs_not_found( self, @@ -614,13 +612,13 @@ async def test_submit_score_set_mappings_to_car_hgvs_not_found( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "not_applicable" + assert event.reason == "no_hgvs" async def test_submit_score_set_mappings_to_car_propagates_exception( self, @@ -724,13 +722,13 @@ async def test_submit_score_set_mappings_to_car_success( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_type == "clingen_allele_id" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.allele_id is not None async def test_submit_score_set_mappings_to_car_preexisting( self, @@ -780,14 +778,14 @@ async def test_submit_score_set_mappings_to_car_preexisting( assert result.data["already_registered_allele_count"] == 1 assert result.data["submitted_allele_count"] == 0 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_metadata["registration_source"] == "preexisting" - assert ann.annotation_metadata["clingen_allele_id"] == "CA_PRIOR" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "preexisting" + assert event.event_metadata["clingen_allele_id"] == "CA_PRIOR" async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( self, @@ -843,13 +841,13 @@ async def test_submit_score_set_mappings_to_car_force_reregister_same_caid( assert result.data["registered_allele_count"] == 1 assert result.data["submitted_allele_count"] == 1 - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" - assert ann.annotation_metadata["registration_source"] == "reconfirmed" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" + assert event.reason == "reconfirmed" async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( self, @@ -909,14 +907,15 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( session.refresh(allele) assert allele.clingen_allele_id == "CA_STORED" - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "failed" - assert ann.annotation_metadata["clingen_allele_id"] == "CA_STORED" - assert ann.annotation_metadata["conflicting_caid"] == "CA_DIFFERENT" + assert len(events) == 1 + for event in events: + assert event.disposition == "failed" + assert event.reason == "caid_conflict" + assert event.event_metadata["clingen_allele_id"] == "CA_STORED" + assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" @pytest.mark.integration @@ -979,12 +978,12 @@ async def test_submit_score_set_mappings_to_car_independent_ctx( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1046,12 +1045,12 @@ async def test_submit_score_set_mappings_to_car_pipeline_ctx( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run_in_pipeline) @@ -1090,8 +1089,8 @@ async def test_submit_score_set_mappings_to_car_submission_disabled( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1129,8 +1128,8 @@ async def test_submit_score_set_mappings_to_car_no_submission_endpoint( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1160,8 +1159,8 @@ async def test_submit_score_set_mappings_to_car_no_mappings( assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars(select(VariantAnnotationStatus)).all() - assert len(annotation_statuses) == 0 + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 0 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1214,10 +1213,10 @@ async def test_submit_score_set_mappings_to_car_no_registered_alleles( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1274,10 +1273,10 @@ async def test_submit_score_set_mappings_to_car_no_linked_alleles( assert len(alleles) == 0 # Verify annotation statuses were rendered as failed — 4 variants, all failed - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 + assert len(events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1344,13 +1343,13 @@ async def test_submit_score_set_mappings_to_car_partial_failure( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # All 4 variant annotations succeeded - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.status == "success", + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.disposition == "present", ) ).all() - assert len(success_annotations) == 4 + assert len(present_events) == 1 # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_car_sample_job_run) @@ -1409,18 +1408,19 @@ async def test_submit_score_set_mappings_to_car_car_error_details_stored_in_anno standalone_worker_context, submit_score_set_mappings_to_car_sample_job_run.id ) - # All 4 variant annotations should have EXTERNAL_SERVICE_REJECTED since the 1 shared allele was rejected - car_rejected_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == "clingen_allele_id", - VariantAnnotationStatus.failure_category == "external_service_rejected", + # The 1 shared allele was rejected by ClinGen → one failed event (reason service_rejected). + rejected_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == "clingen_allele_id", + AnnotationEvent.reason == "service_rejected", ) ).all() - assert len(car_rejected_annotations) == 4 - for rejected in car_rejected_annotations: - assert rejected.annotation_metadata["submitted_hgvs"] == allele_hgvs - assert rejected.annotation_metadata["car_error_type"] == "InvalidHGVS" - assert rejected.annotation_metadata["car_error_message"] == "The HGVS string is invalid." + assert len(rejected_events) == 1 + for event in rejected_events: + assert event.disposition == "failed" + assert event.event_metadata["submitted_hgvs"] == allele_hgvs + assert event.event_metadata["car_error_type"] == "InvalidHGVS" + assert event.event_metadata["car_error_message"] == "The HGVS string is invalid." async def test_submit_score_set_mappings_to_car_propagates_exception_to_decorator( self, @@ -1536,12 +1536,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_independent( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( self, @@ -1608,12 +1608,12 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_pipeline( assert alleles_with_caid[0].clingen_allele_id == f"CA{alleles[0].id}" # 4 per-variant annotations — all success - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 4 - for ann in annotation_statuses: - assert ann.status == "success" + assert len(events) == 1 + for event in events: + assert event.disposition == "present" async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_independent( self, @@ -1668,10 +1668,10 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handling_pipeline( self, @@ -1731,10 +1731,10 @@ async def test_submit_score_set_mappings_to_car_with_arq_context_exception_handl assert len(alleles) == 0 # Verify no annotation statuses were created - annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "clingen_allele_id") + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") ).all() - assert len(annotation_statuses) == 0 + assert len(events) == 0 @pytest.mark.unit @@ -2084,11 +2084,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2155,11 +2155,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2262,11 +2262,11 @@ async def dummy_no_linked_alleles_submission(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2310,7 +2310,7 @@ async def test_submit_score_set_mappings_to_ldh_hgvs_not_found( # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2365,11 +2365,11 @@ async def dummy_submission_failure(*args, **kwargs): # Verify annotation statuses were created with failures annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "failed" + assert ann.disposition == "failed" # Verify the job status is updated in the database # TODO:XXX: Change status to 'failed' once decorator supports it @@ -2435,15 +2435,15 @@ async def dummy_partial_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 success_count = 0 failure_count = 0 for ann in annotation_statuses: - if ann.status == "success": + if ann.disposition == "present": success_count += 1 - elif ann.status == "failed": + elif ann.disposition == "failed": failure_count += 1 assert success_count == 1 @@ -2513,11 +2513,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2590,11 +2590,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run) @@ -2662,11 +2662,11 @@ async def dummy_ldh_submission(*args, **kwargs): # Verify annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 4 for ann in annotation_statuses: - assert ann.status == "success" + assert ann.disposition == "present" # Verify the job status is updated in the database session.refresh(submit_score_set_mappings_to_ldh_sample_job_run_in_pipeline) @@ -2721,7 +2721,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 @@ -2776,7 +2776,7 @@ async def test_submit_score_set_mappings_to_ldh_with_arq_context_exception_handl mock_send_slack_job_error.assert_called_once() # Verify no annotation statuses were created annotation_statuses = session.scalars( - select(VariantAnnotationStatus).where(VariantAnnotationStatus.annotation_type == "ldh_submission") + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "ldh_submission") ).all() assert len(annotation_statuses) == 0 From 73007fd54a2527638200c212d106f4b52e93f4b9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:05 -0700 Subject: [PATCH 36/93] feat(clinvar): emit annotation events from ClinVar job The ClinVar job records AnnotationEvents via record_event, distinguishing superseded/no_record/no_caid/multi_variant_caid cases through the shared EventReason vocabulary. Clarifies the ClinicalControl comments to mark the row-level vs variation-level link identifiers. --- src/mavedb/models/clinical_control.py | 4 +- .../worker/jobs/external_services/clinvar.py | 197 +++++++++--------- .../external_services/network/test_clinvar.py | 37 ++-- .../jobs/external_services/test_clinvar.py | 122 ++++++----- 4 files changed, 202 insertions(+), 158 deletions(-) diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index cf28b5b8b..694aad58e 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -28,11 +28,11 @@ class ClinvarControl(Base): clinical_review_status = Column(String, nullable=False) db_name = Column(String, nullable=False, index=True) - # Currently the ClinVar Allele ID. + # ClinVar Allele ID (row level link). db_identifier = Column(String, nullable=False, index=True) db_version = Column(String, nullable=False, index=True) - # ClinVar's canonical public identifier (ClinVar Variation ID). + # ClinVar Variation ID (variation level link). clinvar_variation_id = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 8041d14f7..ad9eaf654 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -12,10 +12,11 @@ """ import logging +from collections import Counter from datetime import datetime import requests -from sqlalchemy import func, select +from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert as pg_insert from mavedb.lib.annotation_status_manager import AnnotationStatusManager @@ -26,7 +27,9 @@ from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.score_set import ScoreSet from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management @@ -47,46 +50,40 @@ def _generate_clinvar_versions() -> list[tuple[int, int]]: snapshot that should be processed. """ current_year = datetime.now().year - versions = [(CLINVAR_START_YEAR, CLINVAR_START_MONTH)] - for year in range(CLINVAR_START_YEAR + 1, current_year + 1): - versions.append((year, 1)) - return versions + first_version = (CLINVAR_START_YEAR, CLINVAR_START_MONTH) + archival_versions = [(year, 1) for year in range(CLINVAR_START_YEAR + 1, current_year + 1)] + return [first_version, *archival_versions] def _annotate_clinvar( annotation_manager: AnnotationStatusManager, - variant_ids: list[int], - version: str, - status: AnnotationStatus, + allele_id: int, + disposition: Disposition, + reason: EventReason, *, - failure_category: AnnotationFailureCategory | None = None, + source_version: str, error_message: str | None = None, metadata: dict | None = None, ) -> None: - """Fan a CLINVAR_CONTROL annotation out to every variant served by an allele. + """Record one CLINVAR_CONTROL event for an allele at a given ClinVar release. - AAS migration seam: the single choke point for ClinVar's per-variant VAS writes. At migration it - becomes an allele-keyed event writer; the per-variant fan-out goes away, and the variant - association narrows to provenance. See docs/design/allele-annotation-status.md. - - Unlike gnomAD/VEP, ClinVar writes a status row per ClinVar release, so retirement is - version-scoped (``replace_all_versions=False``) — each release keeps its own current row. + The single choke point for ClinVar's status writes. One event per (allele, release); provenance + (which variants drove the linkage) is derived from the live links as-of the event, not fanned out + here. Unlike gnomAD/VEP, ClinVar is multi-version, so ``source_version`` (the ClinVar release) + distinguishes an allele's events across releases. """ - annotation_data: dict = {"annotation_metadata": metadata or {}} + meta = dict(metadata or {}) if error_message is not None: - annotation_data["error_message"] = error_message - - for variant_id in variant_ids: - annotation_manager.add_annotation( - variant_id=variant_id, - annotation_type=AnnotationType.CLINVAR_CONTROL, - version=version, - status=status, - failure_category=failure_category, - annotation_data=annotation_data, - current=True, - replace_all_versions=False, - ) + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CLINVAR_CONTROL, + allele_id=allele_id, + disposition=disposition, + reason=reason, + source_version=source_version, + metadata=meta or None, + ) @with_pipeline_management @@ -139,13 +136,23 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often - # the RT-derived one; VAS still fans only to authoritative_variant_ids (the bandaid seam). + # the RT-derived one. Events are allele-keyed, so every allele is recorded per release (no fan-out). allele_data = group_alleles_for_annotation( get_alleles_for_score_set(job_manager.db, score_set.id), payload=lambda row: row.clingen_allele_id, ) job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) + # Link counts accumulate across all versions (an allele may link in every release it appears in). + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) + if not allele_data: logger.warning( msg="No current alleles with CAIDs were found for this score set. Skipping ClinVar refresh.", @@ -153,12 +160,7 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": 0, - "versions_total": len(versions), - "created_link_count": 0, - "preexisting_link_count": 0, - } + data={"versions_completed": 0, "versions_total": len(versions), **dict(annotation_counts)} ) all_allele_ids = set(allele_data.keys()) @@ -175,11 +177,7 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ).all() ) - total_failed = 0 - created_link_count = 0 - preexisting_link_count = 0 versions_completed = 0 - for version_index, (year, month) in enumerate(versions): clinvar_version = f"{month:02d}_{year}" job_manager.save_to_context({"current_version": clinvar_version, "version_index": version_index}) @@ -206,28 +204,32 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: # bypasses the skip but the link write still get-or-creates, so a forced no-op writes nothing. already_linked = set() if force else alleles_linked_at_version(clinvar_version) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) - for allele_id, entry in allele_data.items(): - caid = entry.payload - + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id + ) + for allele_id, caid in allele_data.items(): if allele_id in already_linked: - preexisting_link_count += 1 + annotation_counts["preexisting_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "action": "preexisting"}, + allele_id, + Disposition.PRESENT, + EventReason.PREEXISTING, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid}, ) continue + # A cis-block (multi-variant) CAID structurally cannot key ClinVar — a terminal gap, not + # a statement about ClinVar's contents. if "," in caid: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.MULTI_VARIANT_CAID, + source_version=clinvar_version, error_message="Multi-variant ClinGen allele IDs cannot be associated with ClinVar data.", metadata={"clingen_allele_id": caid}, ) @@ -236,13 +238,13 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: try: clinvar_allele_id = await get_associated_clinvar_allele_id(caid) except requests.exceptions.RequestException as exc: - total_failed += 1 + annotation_counts["failed_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.EXTERNAL_API_ERROR, + allele_id, + Disposition.FAILED, + EventReason.API_ERROR, + source_version=clinvar_version, error_message=f"Failed to retrieve ClinVar allele ID from ClinGen API: {str(exc)}", metadata={"clingen_allele_id": caid}, ) @@ -253,27 +255,33 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ) continue + # ClinGen has no ClinVar AlleleID for this CAID — the allele is not a ClinVar control: an + # informative negative about the source, not a pipeline gap. if not clinvar_allele_id: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.NO_LINKED_ALLELE, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, error_message="No ClinVar allele ID found for ClinGen allele ID.", metadata={"clingen_allele_id": caid}, ) continue + # The allele has a ClinVar AlleleID but it is absent from this release's snapshot — a + # genuine, version-scoped negative (ClinVar queried, nothing for this release). if clinvar_allele_id not in tsv_data: + annotation_counts["skipped_link_count"] += 1 _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SKIPPED, - failure_category=AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND, + allele_id, + Disposition.ABSENT, + EventReason.NO_RECORD, + source_version=clinvar_version, error_message="No ClinVar data found for ClinVar allele ID.", - metadata={"clingen_allele_id": caid}, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) continue @@ -323,20 +331,19 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: ) if live_link is None: job_manager.db.add(ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id)) - created_link_count += 1 - action = "created" + annotation_counts["created_link_count"] += 1 + reason = EventReason.CREATED elif live_link.clinvar_control_id == clinvar_control.id: - preexisting_link_count += 1 - action = "preexisting" + annotation_counts["preexisting_link_count"] += 1 + reason = EventReason.PREEXISTING else: - at = job_manager.db.scalar(select(func.now())) - assert at is not None # SELECT now() always returns a timestamp - live_link.retire(at=at) + retired_at = datetime.now() + live_link.retire(at=retired_at) job_manager.db.add( - ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=at) + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=retired_at) ) - created_link_count += 1 - action = "superseded" + annotation_counts["created_link_count"] += 1 + reason = EventReason.SUPERSEDED logger.warning( msg=( f"Allele {allele_id} held a live ClinVar link to control " @@ -350,10 +357,11 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: _annotate_clinvar( annotation_manager, - entry.authoritative_variant_ids, - clinvar_version, - AnnotationStatus.SUCCESS, - metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id, "action": action}, + allele_id, + Disposition.PRESENT, + reason, + source_version=clinvar_version, + metadata={"clingen_allele_id": caid, "clinvar_allele_id": clinvar_allele_id}, ) annotation_manager.flush() @@ -366,14 +374,17 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: logger.info( f"ClinVar refresh complete: {versions_completed}/{len(versions)} versions, " - f"{created_link_count} new links, {preexisting_link_count} preexisting.", + f"{annotation_counts['created_link_count']} new links, " + f"{annotation_counts['preexisting_link_count']} preexisting.", extra=job_manager.logging_context(), ) - if total_failed > 0 and created_link_count == 0 and preexisting_link_count == 0: - error_message = ( - f"All {total_failed} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." - ) + if ( + annotation_counts["failed_link_count"] > 0 + and annotation_counts["created_link_count"] == 0 + and annotation_counts["preexisting_link_count"] == 0 + ): + error_message = f"All {annotation_counts['failed_link_count']} ClinVar lookups failed for score set {score_set.urn}. Possible ClinGen API outage." logger.error(error_message, extra=job_manager.logging_context()) job_manager.db.flush() return JobExecutionOutcome.failed( @@ -381,18 +392,12 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: data={ "versions_completed": versions_completed, "versions_total": len(versions), - "created_link_count": 0, - "preexisting_link_count": 0, + **dict(annotation_counts), }, failure_category=FailureCategory.DEPENDENCY_FAILURE, ) job_manager.db.flush() return JobExecutionOutcome.succeeded( - data={ - "versions_completed": versions_completed, - "versions_total": len(versions), - "created_link_count": created_link_count, - "preexisting_link_count": preexisting_link_count, - } + data={"versions_completed": versions_completed, "versions_total": len(versions), **dict(annotation_counts)} ) diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index 4569d08f7..6d3c0b819 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -10,10 +10,11 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import fetch_clinvar_variant_data +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinical_control import ClinvarControl from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationStatus, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import JobStatus from mavedb.worker.jobs.external_services.clinvar import _generate_clinvar_versions pytestmark = pytest.mark.usefixtures("patch_db_session_ctxmgr") @@ -71,22 +72,32 @@ async def test_refresh_clinvar_controls_e2e( assert len(clinical_controls) >= 1 assert all(cc.db_identifier == "3045425" for cc in clinical_controls) - success_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, - VariantAnnotationStatus.status == AnnotationStatus.SUCCESS, + # Verify that at least one present event was recorded for the allele. The job processes one + # event per ClinVar version; versions without the allele produce absent events, so filtering + # for present gives a stable assertion. Events are allele-keyed (no variant_id). + present_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.PRESENT, ) ).all() - assert len(success_annotations) == 1 - - # Total annotations == versions processed. - all_annotations = session.scalars( - select(VariantAnnotationStatus).where( - VariantAnnotationStatus.annotation_type == AnnotationType.CLINVAR_CONTROL, + assert len(present_events) >= 1 + assert all(e.variant_id is None and e.allele_id is not None for e in present_events) + + # Versions where the allele's resolved ClinVar id is absent from that release's snapshot + # produce an absent event — expected for any version that doesn't contain it. + absent_events = session.scalars( + select(AnnotationEvent).where( + AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, + AnnotationEvent.disposition == Disposition.ABSENT, ) ).all() - assert len(all_annotations) == len(_E2E_VERSIONS) + assert len(absent_events) >= 1 + + # Total events should equal the number of ClinVar versions processed (one allele, one per version). + assert len(present_events) + len(absent_events) == len(_generate_clinvar_versions()) + # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 6b5011236..b757730f9 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -13,8 +13,10 @@ from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory, JobStatus -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager @@ -67,7 +69,7 @@ async def test_no_alleles_with_caids( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarControl)).all() == [] assert session.scalars(select(ClinvarAlleleLink)).all() == [] - assert session.query(VariantAnnotationStatus).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] async def test_fetch_failure_skips_version( self, @@ -105,7 +107,7 @@ async def test_multi_variant_caid_skipped( setup_sample_alleles_with_caid, ): """An allele whose CAID is a multi-variant identifier is skipped (ClinVar can't key on it).""" - variant, allele = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid allele.clingen_allele_id = "CA-MULTI-001,CA-MULTI-002" session.commit() @@ -122,10 +124,13 @@ async def test_multi_variant_caid_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.annotation_type == AnnotationType.CLINVAR_CONTROL - assert vas.failure_category == AnnotationFailureCategory.UNSUPPORTED_IDENTIFIER + # Allele-keyed event: a cis-block CAID structurally can't key ClinVar — a pipeline gap, not a + # statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.MULTI_VARIANT_CAID + assert event.allele_id == allele.id and event.variant_id is None async def test_no_associated_clinvar_allele_id_skipped( self, @@ -135,8 +140,8 @@ async def test_no_associated_clinvar_allele_id_skipped( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """ClinGen returns no ClinVar allele id for the CAID -> SKIPPED/NO_LINKED_ALLELE, no link.""" - variant, _ = setup_sample_alleles_with_caid + """ClinGen returns no ClinVar allele id for the CAID -> absent/no_record event, no link.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -156,9 +161,11 @@ async def test_no_associated_clinvar_allele_id_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.failure_category == AnnotationFailureCategory.NO_LINKED_ALLELE + # ClinGen had no ClinVar AlleleID for this CAID — an informative negative about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None async def test_clinvar_data_not_found_skipped( self, @@ -168,8 +175,8 @@ async def test_clinvar_data_not_found_skipped( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """The resolved ClinVar allele id is absent from the version's TSV -> SKIPPED, no link.""" - variant, _ = setup_sample_alleles_with_caid + """The resolved ClinVar allele id is absent from the version's TSV -> absent/no_record, no link.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -189,9 +196,12 @@ async def test_clinvar_data_not_found_skipped( assert result.status == JobStatus.SUCCEEDED assert session.scalars(select(ClinvarAlleleLink)).all() == [] - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SKIPPED - assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_REFERENCE_NOT_FOUND + # The CAID resolved to a ClinVar AlleleID, but it's absent from this release's snapshot — a + # genuine, version-scoped negative. + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.ABSENT + assert event.reason == EventReason.NO_RECORD + assert event.allele_id == allele.id and event.variant_id is None async def test_clingen_api_failure_fails_when_total( self, @@ -202,7 +212,7 @@ async def test_clingen_api_failure_fails_when_total( setup_sample_alleles_with_caid, ): """A ClinGen API error with no successful links returns FAILED/DEPENDENCY_FAILURE.""" - variant, _ = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -222,9 +232,10 @@ async def test_clingen_api_failure_fails_when_total( assert result.status == JobStatus.FAILED assert result.failure_category == FailureCategory.DEPENDENCY_FAILURE - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.FAILED - assert vas.failure_category == AnnotationFailureCategory.EXTERNAL_API_ERROR + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.allele_id == allele.id and event.variant_id is None async def test_successful_link_new_control( self, @@ -234,8 +245,8 @@ async def test_successful_link_new_control( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """A resolved + present CAID creates a ClinvarControl, a live link, and a SUCCESS VAS.""" - variant, allele = setup_sample_alleles_with_caid + """A resolved + present CAID creates a ClinvarControl, a live link, and a present/created event.""" + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -269,11 +280,13 @@ async def test_successful_link_new_control( assert len(links) == 1 assert links[0].clinvar_control_id == control.id - vas = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).one() - assert vas.status == AnnotationStatus.SUCCESS - assert vas.annotation_metadata["action"] == "created" + event = session.scalars(select(AnnotationEvent)).one() + assert event.disposition == Disposition.PRESENT + assert event.reason == EventReason.CREATED + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.allele_id == allele.id and event.variant_id is None - async def test_links_rt_derived_allele_but_annotates_only_authoritative( + async def test_links_and_annotates_full_allele_set( self, mock_worker_ctx, session, @@ -281,10 +294,13 @@ async def test_links_rt_derived_allele_but_annotates_only_authoritative( sample_refresh_clinvar_controls_job_run, setup_rt_derived_allele_with_caid, ): - """ClinVar linkage covers the full allele set: the RT-derived allele carries the matching CAID - and is linked, while per-variant VAS is written only for the authoritative link (the bandaid). + """ClinVar linkage covers the full allele set, not just authoritative links: the RT-derived + allele carries the matching CAID and is linked. Events are now allele-keyed, so the RT-derived + allele's present status is recorded directly — the limitation the per-variant bandaid had + (dropping the RT-derived allele's status) is lifted. Each allele gets its own event: present for + the linked RT allele, absent for the unmatched authoritative one. """ - variant, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid + _, authoritative_allele, rt_allele = setup_rt_derived_allele_with_caid rt_caid = rt_allele.clingen_allele_id async def resolve(caid): @@ -322,11 +338,17 @@ async def resolve(caid): == [] ) - # VAS is written only for the authoritative link: exactly one row, keyed to the variant. - statuses = session.query(VariantAnnotationStatus).all() - assert len(statuses) == 1 - assert statuses[0].variant_id == variant.id - assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + # One allele-keyed event per allele (never per-variant): the linked RT allele is present, the + # unmatched authoritative allele is absent. The variant resolves its derived allele's status + # through the live links — no variant_id on the events. + events = session.scalars(select(AnnotationEvent)).all() + assert all(e.annotation_type == AnnotationType.CLINVAR_CONTROL for e in events) + assert all(e.variant_id is None for e in events) + by_allele = {e.allele_id: e for e in events} + assert by_allele[rt_allele.id].disposition == Disposition.PRESENT + assert by_allele[rt_allele.id].reason == EventReason.CREATED + assert by_allele[authoritative_allele.id].disposition == Disposition.ABSENT + assert by_allele[authoritative_allele.id].reason == EventReason.NO_RECORD async def test_idempotent_rerun_skips_and_does_not_duplicate( self, @@ -338,7 +360,7 @@ async def test_idempotent_rerun_skips_and_does_not_duplicate( ): """A second run finds the allele already linked at the version, skips the resolution, reports preexisting, and creates neither a duplicate control nor a duplicate link.""" - variant, allele = setup_sample_alleles_with_caid + _, allele = setup_sample_alleles_with_caid with ( patch( @@ -376,10 +398,15 @@ async def test_idempotent_rerun_skips_and_does_not_duplicate( assert len(links) == 1 assert links[0].valid_to is None - # Per-version VAS: two rows for the variant, only one current. - statuses = session.query(VariantAnnotationStatus).filter_by(variant_id=variant.id).all() - assert len(statuses) == 2 - assert len([s for s in statuses if s.current]) == 1 + # Per-version events: two allele-keyed events (one per run), no current flag. The first run + # created the link; the second found it preexisting. Latest event (by id) wins. + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.allele_id == allele.id).order_by(AnnotationEvent.id) + ).all() + assert len(events) == 2 + assert events[0].reason == EventReason.CREATED + assert events[1].reason == EventReason.PREEXISTING + assert all(e.disposition == Disposition.PRESENT for e in events) async def test_release_reresolution_supersedes_newest_wins( self, @@ -522,7 +549,7 @@ async def test_arq_context_successful_link( sample_refresh_clinvar_controls_job_run, setup_sample_alleles_with_caid, ): - """The job links an allele and records a SUCCESS annotation under an ARQ worker.""" + """The job links an allele and records a present event under an ARQ worker.""" with ( patch( "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", @@ -540,10 +567,11 @@ async def test_arq_context_successful_link( assert len(session.scalars(select(ClinvarControl)).all()) >= 1 assert len(session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.current)).all()) == 1 - statuses = session.query(VariantAnnotationStatus).all() - assert len(statuses) == 1 - assert statuses[0].status == AnnotationStatus.SUCCESS - assert statuses[0].annotation_type == AnnotationType.CLINVAR_CONTROL + events = session.scalars(select(AnnotationEvent)).all() + assert len(events) == 1 + assert events[0].disposition == Disposition.PRESENT + assert events[0].annotation_type == AnnotationType.CLINVAR_CONTROL + assert events[0].allele_id is not None and events[0].variant_id is None session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED @@ -576,7 +604,7 @@ async def test_arq_context_exception_handling( mock_slack.assert_called_once() assert session.scalars(select(ClinvarAlleleLink)).all() == [] - assert session.query(VariantAnnotationStatus).all() == [] + assert session.scalars(select(AnnotationEvent)).all() == [] session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.ERRORED From e31363b1df6633525ebe4bfa52d211a25c678189 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:12 -0700 Subject: [PATCH 37/93] feat(mapping): emit annotation events The mapping job records AnnotationEvents via record_event, deriving disposition and reason from the typed MappingOutcome (mapped / intronic / no_protein_consequence / failed) rather than writing per-variant status rows. --- .../worker/jobs/variant_processing/mapping.py | 46 +++-- .../jobs/variant_processing/test_mapping.py | 166 ++++++++---------- 2 files changed, 97 insertions(+), 115 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index 9e6a8da74..b2f588b72 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -31,7 +31,8 @@ from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -268,7 +269,8 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan f"Processing {total_variants} mapped variants for score set {score_set.urn}.", extra=job_manager.logging_context(), ) - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id) + + annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job.id, score_set_id=score_set.id) for mapped_score in mapped_scores: variant_urn = mapped_score.get("mavedb_id") variant = job_manager.db.scalars(select(Variant).where(Variant.urn == variant_urn)).one() @@ -349,32 +351,28 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan else: job_manager.db.add(mapping_record) - # MAPPED -> success; benign absences -> skipped; FAILED -> failed. The raw outcome - # is preserved in annotation_metadata so the benign distinction survives. + # The mapper emits a MappingRecord for EVERY variant, so "a record exists" carries no + # signal — the signal is whether a real allele resulted. MAPPED yields an authoritative + # allele -> present. A benign absence (intronic, synonymous) produces a record but no + # allele: an informative biological negative -> absent. FAILED -> failed. `reason` reuses + # the MappingOutcome vocabulary so the intronic-vs-no-protein distinction survives. if outcome is MappingOutcome.MAPPED: - annotation_status = AnnotationStatus.SUCCESS - annotation_failure_category = None + disposition = Disposition.PRESENT elif outcome.is_benign_absence: - annotation_status = AnnotationStatus.SKIPPED - annotation_failure_category = None + disposition = Disposition.ABSENT else: - annotation_status = AnnotationStatus.FAILED - annotation_failure_category = AnnotationFailureCategory.EXTERNAL_SERVICE_REJECTED - - annotation_manager.add_annotation( - variant_id=variant.id, # type: ignore - annotation_type=AnnotationType.VRS_MAPPING, - version=tool_version, - status=annotation_status, - failure_category=annotation_failure_category, - annotation_data={ - "error_message": mapped_score.get("error_message", null()), - "annotation_metadata": { - "outcome": outcome.value, - "mapped_assay_level_hgvs": assay_level_hgvs, - }, + disposition = Disposition.FAILED + + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=disposition, + reason=outcome.value, + source_version=tool_version, + metadata={ + "mapped_assay_level_hgvs": assay_level_hgvs, + "error_message": mapped_score.get("error_message"), }, - current=True, ) # Only variants with a post-mapped representation yield an authoritative Allele; diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 3a0c6db2f..96321d816 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -20,7 +20,7 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.lib.managers.job_manager import JobManager from tests.helpers.constants import TEST_CODING_LAYER, TEST_GENOMIC_LAYER, TEST_PROTEIN_LAYER @@ -90,8 +90,8 @@ async def test_map_variants_for_score_set_no_mapping_results( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -137,8 +137,8 @@ async def test_map_variants_for_score_set_no_mapped_scores( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -184,8 +184,8 @@ async def test_map_variants_for_score_set_no_reference_data( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -239,8 +239,8 @@ async def test_map_variants_for_score_set_nonexistent_target_gene( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -286,8 +286,8 @@ async def test_map_variants_for_score_set_returns_variants_not_in_score_set( # Verify no annotations were created annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -354,14 +354,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" @pytest.mark.parametrize( "with_layers", @@ -471,14 +471,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "success" + assert annotation_statuses[0].disposition == "present" async def test_persists_cdna_target_gene_mapping_with_reference_accession_and_null_qc( self, @@ -584,14 +584,14 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 1 assert annotation_statuses[0].annotation_type == "vrs_mapping" - assert annotation_statuses[0].status == "failed" + assert annotation_statuses[0].disposition == "failed" async def test_map_variants_for_score_set_incomplete_mapping( self, @@ -668,17 +668,17 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_status_success = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "success") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "present") .all() ) assert len(annotation_status_success) == 1 assert annotation_status_success[0].annotation_type == "vrs_mapping" annotation_status_failed = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) - .filter(Variant.score_set_id == sample_score_set.id, VariantAnnotationStatus.status == "failed") + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) + .filter(Variant.score_set_id == sample_score_set.id, AnnotationEvent.disposition == "failed") .all() ) assert len(annotation_status_failed) == 1 @@ -694,7 +694,8 @@ async def test_map_variants_for_score_set_benign_outcomes_are_not_failures( ): """A score set whose only unmapped variants are benign absences (intronic / no protein consequence) is ``complete``, not ``incomplete``/``failed``: benign - outcomes carry no allele but are skips, not failures.""" + outcomes carry no allele, so they are recorded as ``absent`` (an informative + biological negative), never ``failed``.""" async def dummy_mapping_job(): mapping_output = await construct_mock_mapping_output( @@ -761,17 +762,17 @@ async def dummy_mapping_job(): assert len(mapping_records) == 2 assert all(authoritative_allele_for(session, r) is None for r in mapping_records) - # Benign outcomes are SKIPPED (not FAILED), and the finer outcome is preserved in metadata. - annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + # Benign outcomes carry no allele → recorded as `absent` (not `failed`), with the finer + # outcome preserved as the event `reason`. + events = ( + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) - assert len(annotation_statuses) == 2 - assert all(s.status == "skipped" for s in annotation_statuses) - assert all(s.failure_category is None for s in annotation_statuses) - recorded_outcomes = {s.annotation_metadata["outcome"] for s in annotation_statuses} + assert len(events) == 2 + assert all(e.disposition == "absent" for e in events) + recorded_outcomes = {e.reason for e in events} assert recorded_outcomes == {"intronic", "no_protein_consequence"} async def test_map_variants_for_score_set_complete_mapping( @@ -850,15 +851,15 @@ async def dummy_mapping_job(): # Verify that annotation statuses were created and correct annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) assert len(annotation_statuses) == 2 for status in annotation_statuses: assert status.annotation_type == "vrs_mapping" - assert status.status == "success" + assert status.disposition == "present" async def test_map_variants_for_score_set_updates_existing_mapped_variants( self, @@ -909,10 +910,10 @@ async def dummy_mapping_job(): ) session.add(prior_link) session.commit() - variant_annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(variant_annotation_status) + session.add(prior_event) session.commit() with ( @@ -965,24 +966,14 @@ async def dummy_mapping_job(): assert new_mapping_record.mapped_date != date(2023, 1, 1) assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify the non-current annotation status still exists - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter( - VariantAnnotationStatus.variant_id == non_current_mapping_record.variant_id, - VariantAnnotationStatus.current.is_(False), - ) - .one_or_none() - ) - assert old_annotation_status is not None - - # Verify that a new annotation status was created - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + # Append-only: the prior event is retained and the re-map appends a new one, so the variant + # now has two vrs_mapping events (there is no current flag to flip). + events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(events) == 2 @pytest.mark.integration @@ -1065,8 +1056,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1156,8 +1147,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1237,7 +1228,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1314,7 +1305,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1390,7 +1381,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1434,10 +1425,10 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) - annotation_status = VariantAnnotationStatus( - variant_id=variant.id, current=True, annotation_type="vrs_mapping", status="success" + prior_event = AnnotationEvent( + variant_id=variant.id, annotation_type="vrs_mapping", disposition="present", reason="mapped" ) - session.add(annotation_status) + session.add(prior_event) session.add(mapping_record) session.commit() @@ -1496,23 +1487,16 @@ async def dummy_mapping_job(): assert new_mapping_record.mapped_date != date(2023, 1, 1) assert new_mapping_record.mapping_api_version != "v1.0.0" - # Verify that annotation statuses where marked as non-current and new entries created - annotation_statuses = session.query(VariantAnnotationStatus).all() - assert len(annotation_statuses) == len(variants) * 2 # Each variant has two annotation statuses now + # Append-only: each variant keeps its prior event and gains a new one from this run. + annotation_events = session.query(AnnotationEvent).all() + assert len(annotation_events) == len(variants) * 2 for variant in variants: - old_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(False)) - .one_or_none() - ) - assert old_annotation_status is not None - - new_annotation_status = ( - session.query(VariantAnnotationStatus) - .filter(VariantAnnotationStatus.variant_id == variant.id, VariantAnnotationStatus.current.is_(True)) - .one_or_none() + variant_events = ( + session.query(AnnotationEvent) + .filter(AnnotationEvent.variant_id == variant.id, AnnotationEvent.annotation_type == "vrs_mapping") + .all() ) - assert new_annotation_status is not None + assert len(variant_events) == 2 # Verify that the job status was updated. processing_run = ( @@ -1573,7 +1557,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1632,7 +1616,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1716,8 +1700,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1804,8 +1788,8 @@ async def dummy_mapping_job(): # Verify that each variant has an annotation status annotation_statuses = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter(Variant.score_set_id == sample_score_set.id) .all() ) @@ -1874,7 +1858,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. @@ -1928,7 +1912,7 @@ async def dummy_mapping_job(): assert len(mapping_records) == 0 # Verify that no annotation statuses were created - annotation_statuses = session.query(VariantAnnotationStatus).all() + annotation_statuses = session.query(AnnotationEvent).all() assert len(annotation_statuses) == 0 # Verify that the job status was updated. From 6288f82b0bf4a36881f913be538d45e4e46ea91d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:19 -0700 Subject: [PATCH 38/93] feat(reverse-translation): emit annotation events The reverse-translation job records AnnotationEvents via record_event, using the RT-specific EventReason cases (translated / no_coding_transcript / no_assay_level_hgvs / translation_failed / translation_error / transcript_unresolved) to distinguish informative negatives from failures. --- .../variant_processing/reverse_translation.py | 179 +++++++++--------- .../test_reverse_translation.py | 91 +++++---- 2 files changed, 141 insertions(+), 129 deletions(-) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 07df529fd..d7f17e919 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -11,12 +11,13 @@ import dataclasses import functools import logging +from collections import Counter from datetime import date -from enum import Enum from typing import Any, NamedTuple, Sequence from ga4gh.vrs.extras.translator import AlleleTranslator from sqlalchemy import select +from variant_annotation import __version__ as variant_annotation_version from variant_annotation.lib.accessions import looks_like_refseq_protein_accession from variant_annotation.lib.translation import construct_equivalent_variants from variant_annotation.lib.translation.types import TranslationConfig, VariantInput, WtCodonMode @@ -29,7 +30,9 @@ from mavedb.models.allele import Allele as AlleleDbModel from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.enums.annotation_type import AnnotationType -from mavedb.models.enums.job_pipeline import AnnotationFailureCategory, AnnotationStatus, FailureCategory +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.job_pipeline import FailureCategory from mavedb.models.enums.target_category import TargetCategory from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -64,36 +67,21 @@ class _TranscriptResolution(NamedTuple): target_gene_id: int | None -class _TranscriptResolutionSkipReason(Enum): - NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" - TRANSCRIPT_UNRESOLVED = "transcript_unresolved" - NO_CODING_TRANSCRIPT = "no_coding_transcript" - - @classmethod - def classify( - cls, resolution: _TranscriptResolution, category: TargetCategory | None - ) -> tuple["_TranscriptResolutionSkipReason", str]: - """Classify why a record was skipped. - - Protein-coding target with no transcript → recoverable (transcript_unresolved). - Non-coding/regulatory target → correct skip (no_coding_transcript). - """ - if not resolution.rec.hgvs_assay_level: - return ( - cls.NO_ASSAY_LEVEL_HGVS, - "No assay-level HGVS available to reverse-translate.", - ) - if category == TargetCategory.protein_coding: - return ( - cls.TRANSCRIPT_UNRESOLVED, - "Protein-coding target but no coding transcript could be resolved " - "(no cdna TargetGeneMapping and no NP_->NM_ association). Recoverable: " - "re-map or check transcript selection.", - ) - return ( - cls.NO_CODING_TRANSCRIPT, - "Non-coding/regulatory target has no protein consequence to reverse-translate.", - ) +def _classify_skip( + resolution: _TranscriptResolution, category: TargetCategory | None +) -> tuple[EventReason, Disposition]: + """Classify why a record was skipped, returning its reason and event disposition. + + ``no_assay_level_hgvs`` — no input to translate, we could not ask → ``not_applicable``. + ``transcript_unresolved`` — protein-coding target with no transcript, a recoverable pipeline + gap → ``failed``. ``no_coding_transcript`` — non-coding target has no protein consequence, a + biological negative → ``absent``. + """ + if not resolution.rec.hgvs_assay_level: + return (EventReason.NO_ASSAY_LEVEL_HGVS, Disposition.NOT_APPLICABLE) + if category == TargetCategory.protein_coding: + return (EventReason.TRANSCRIPT_UNRESOLVED, Disposition.FAILED) + return (EventReason.NO_CODING_TRANSCRIPT, Disposition.ABSENT) def _coding_transcripts_for_proteins(protein_accessions: set[str]) -> dict[str, str]: @@ -147,6 +135,33 @@ def _build_translation_config(overrides: dict[str, Any] | None) -> TranslationCo raise ValueError(f"Invalid translation_config: {exc}") from exc +def _annotate_translation( + annotation_manager: AnnotationStatusManager, + variant_id: int, + disposition: Disposition, + reason: EventReason, + *, + error_message: str | None = None, + metadata: dict | None = None, +) -> None: + """Record one CROSS_LEVEL_TRANSLATION event for an allele (the translation is a variant-level fact). + + The single choke point for RT's status writes. + """ + meta = dict(metadata or {}) + if error_message is not None: + meta["error_message"] = error_message + + annotation_manager.record_event( + AnnotationType.CROSS_LEVEL_TRANSLATION, + variant_id=variant_id, + disposition=disposition, + reason=reason, + source_version=variant_annotation_version, + metadata=meta or None, + ) + + @with_pipeline_management async def reverse_translate_variants_for_score_set( ctx: dict, job_id: int, job_manager: JobManager @@ -227,13 +242,15 @@ async def reverse_translate_variants_for_score_set( .all() ) + annotation_counts: Counter[str] = Counter({"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + if not rows: logger.warning( msg="No current and authoritative mapping records found for this score set.", extra=job_manager.logging_context(), ) job_manager.db.flush() - return JobExecutionOutcome.succeeded(data={"translated": 0, "failed": 0, "skipped": 0, "alleles_created": 0}) + return JobExecutionOutcome.succeeded(data=dict(annotation_counts)) # Genomic/cdna records resolve via the cdna TargetGeneMapping; protein records have no # cdna reference_accession, so collect their NP_ accessions for a batched NP_→NM_ UTA lookup. @@ -322,10 +339,9 @@ async def reverse_translate_variants_for_score_set( extra=job_manager.logging_context(), ) - translated = 0 - failed = 0 - alleles_created = 0 - annotation_manager = AnnotationStatusManager(job_manager.db, job_run_id=job_manager.job_id) + annotation_manager = AnnotationStatusManager( + job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set_id + ) allele_translator = AlleleTranslator(ctx["seqrepo"]) current_record_ids = ( @@ -411,11 +427,12 @@ async def reverse_translate_variants_for_score_set( ) candidate_count += 1 - alleles_created += candidate_count + annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, "hgvs_c_candidates": result.hgvs_c_candidates, "hgvs_g_candidates": result.hgvs_g_candidates, + "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, } @@ -423,24 +440,23 @@ async def reverse_translate_variants_for_score_set( # No translatable candidates and failures mean the variant failed reverse translation. No # failures and no candidates is a success with no alleles created. if candidate_count == 0 and failed_candidates: - failed += 1 - annotation_manager.add_annotation( + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": "All candidate HGVS failed VRS translation.", - "annotation_metadata": annotation_metadata, - }, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_FAILED, + error_message="All candidate HGVS failed VRS translation.", + metadata=annotation_metadata, ) else: - translated += 1 - annotation_manager.add_annotation( + annotation_counts["translated"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.SUCCESS, - annotation_data={"annotation_metadata": annotation_metadata}, + disposition=Disposition.PRESENT, + reason=EventReason.TRANSLATED, + metadata=annotation_metadata, ) # Supersede prior live derived links atomically. @@ -453,71 +469,56 @@ async def reverse_translate_variants_for_score_set( MappingRecordAllele.mapping_record_id.in_(current_record_ids), ) - # TODO#767: non-substitution consequences (del/ins/delins/fs/ext) have no synonymous + # TODO#767: some non-substitution consequences (del/ins/delins/fs/ext) have no synonymous # equivalence class, so they arrive here as TranslationErrors and are miscounted as # FAILED. Classify by the protein consequence's edit type up front and map these to # SKIPPED instead of pattern-matching engine error strings. for error in errors: _rec, variant = variant_input_map[id(error.input)] - failed += 1 - annotation_manager.add_annotation( + annotation_counts["failed"] += 1 + _annotate_translation( + annotation_manager, variant_id=variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.FAILED, - failure_category=AnnotationFailureCategory.UNKNOWN, - annotation_data={ - "error_message": error.error, - "annotation_metadata": {"hgvs_input": error.input.hgvs}, - }, + disposition=Disposition.FAILED, + reason=EventReason.TRANSLATION_ERROR, + metadata={"hgvs_input": error.input.hgvs, "error_message": error.error}, ) - skipped = len(skipped_variants) + annotation_counts["skipped"] = len(skipped_variants) for p in skipped_variants: category = target_category_by_gene.get(p.target_gene_id) if p.target_gene_id is not None else None - skip_category, reason = _TranscriptResolutionSkipReason.classify(p, category) - annotation_manager.add_annotation( + reason, disposition = _classify_skip(p, category) + _annotate_translation( + annotation_manager, variant_id=p.variant.id, - annotation_type=AnnotationType.CROSS_LEVEL_TRANSLATION, - status=AnnotationStatus.SKIPPED, - annotation_data={ - "annotation_metadata": { - "hgvs_input": p.rec.hgvs_assay_level, - "skip_category": skip_category.value, - "reason": reason, - } - }, + disposition=disposition, + reason=reason, + metadata={"hgvs_input": p.rec.hgvs_assay_level}, ) annotation_manager.flush() - job_manager.save_to_context( - { - "translated": translated, - "failed": failed, - "skipped": skipped, - "alleles_created": alleles_created, - } - ) + outcome_data = dict(annotation_counts) + job_manager.save_to_context(outcome_data) logger.info( msg=( - f"Reverse translation complete: {translated} translated, {failed} failed, " - f"{skipped} skipped, {alleles_created} alleles created." + f"Reverse translation complete: {annotation_counts['translated']} translated, " + f"{annotation_counts['failed']} failed, {annotation_counts['skipped']} skipped, " + f"{annotation_counts['alleles_created']} alleles created." ), extra=job_manager.logging_context(), ) job_manager.db.flush() - if translated == 0 and failed > 0: + if annotation_counts["translated"] == 0 and annotation_counts["failed"] > 0: logger.error( msg="All variant reverse translations failed.", extra=job_manager.logging_context(), ) return JobExecutionOutcome.failed( reason="All variant reverse translations failed.", - data={"translated": 0, "failed": failed, "skipped": skipped, "alleles_created": 0}, + data=outcome_data, failure_category=FailureCategory.DATA_ERROR, ) - return JobExecutionOutcome.succeeded( - data={"translated": translated, "failed": failed, "skipped": skipped, "alleles_created": alleles_created} - ) + return JobExecutionOutcome.succeeded(data=outcome_data) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 970fa8544..1a77e8e86 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -21,7 +21,7 @@ from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.variant_processing.mapping import map_variants_for_score_set from mavedb.worker.jobs.variant_processing.reverse_translation import ( _build_translation_config, @@ -143,17 +143,19 @@ async def _reverse_translate(session, mock_worker_ctx, rt_run): ) -def _cross_level_statuses(session, score_set_id, status=None): +def _cross_level_events(session, score_set_id, disposition=None, reason=None): query = ( - session.query(VariantAnnotationStatus) - .join(Variant, VariantAnnotationStatus.variant_id == Variant.id) + session.query(AnnotationEvent) + .join(Variant, AnnotationEvent.variant_id == Variant.id) .filter( Variant.score_set_id == score_set_id, - VariantAnnotationStatus.annotation_type == "cross_level_translation", + AnnotationEvent.annotation_type == "cross_level_translation", ) ) - if status is not None: - query = query.filter(VariantAnnotationStatus.status == status) + if disposition is not None: + query = query.filter(AnnotationEvent.disposition == disposition) + if reason is not None: + query = query.filter(AnnotationEvent.reason == reason) return query.all() @@ -226,7 +228,7 @@ async def test_no_mapping_records_is_a_noop( # No candidate alleles, links, or annotations were produced. assert session.query(Allele).count() == 0 assert _non_authoritative_links(session) == [] - assert _cross_level_statuses(session, sample_score_set.id) == [] + assert _cross_level_events(session, sample_score_set.id) == [] async def test_creates_genomic_and_coding_candidate_alleles( self, @@ -287,9 +289,9 @@ async def test_creates_genomic_and_coding_candidate_alleles( mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} - statuses = _cross_level_statuses(session, sample_score_set.id) - assert len(statuses) == 1 - assert statuses[0].status == "success" + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 1 + assert events[0].disposition == "present" async def test_transcript_is_queryable_via_derived_expression( self, @@ -604,10 +606,11 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( # The two candidate alleles are reused across runs, not duplicated. assert session.query(Allele).filter(Allele.vrs_digest.in_(["ga4gh:VA.genomic", "ga4gh:VA.coding"])).count() == 2 - # The cross-level translation status is versioned the same way: prior retired, one current. - statuses = _cross_level_statuses(session, sample_score_set.id) - assert len(statuses) == 2 - assert len([s for s in statuses if s.current]) == 1 + # The cross-level translation log is append-only: the rerun adds a second event, and + # "current" is the latest by id (there is no current flag to desync). + events = _cross_level_events(session, sample_score_set.id) + assert len(events) == 2 + assert max(events, key=lambda e: e.id).disposition == "present" async def test_all_failures_fail_the_job( self, @@ -644,11 +647,11 @@ async def test_all_failures_fail_the_job( assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} assert _non_authoritative_links(session) == [] - failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") - assert len(failed_statuses) == 1 - assert failed_statuses[0].error_message == "forward translation failed" + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "forward translation failed" # Library/input-level failures still carry the assay-level HGVS as metadata. - assert failed_statuses[0].annotation_metadata == {"hgvs_input": assay_hgvs} + assert failed_events[0].event_metadata["hgvs_input"] == assay_hgvs async def test_partial_success_succeeds_with_mixed_annotations( self, @@ -696,8 +699,8 @@ async def test_partial_success_succeeds_with_mixed_annotations( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 1, "skipped": 0, "alleles_created": 1} - assert len(_cross_level_statuses(session, sample_score_set.id, status="success")) == 1 - assert len(_cross_level_statuses(session, sample_score_set.id, status="failed")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="present")) == 1 + assert len(_cross_level_events(session, sample_score_set.id, disposition="failed")) == 1 assert len(_non_authoritative_links(session)) == 1 async def test_partial_candidate_translation_failure_keeps_success_with_metadata( @@ -742,9 +745,9 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} assert len(_non_authoritative_links(session)) == 1 - statuses = _cross_level_statuses(session, sample_score_set.id, status="success") - assert len(statuses) == 1 - failed_candidates = statuses[0].annotation_metadata["failed_candidates"] + events = _cross_level_events(session, sample_score_set.id, disposition="present") + assert len(events) == 1 + failed_candidates = events[0].event_metadata["failed_candidates"] assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] async def test_all_candidates_failing_translation_marks_variant_failed( @@ -785,16 +788,16 @@ async def test_all_candidates_failing_translation_marks_variant_failed( assert result.data == {"translated": 0, "failed": 1, "skipped": 0, "alleles_created": 0} assert _non_authoritative_links(session) == [] - failed_statuses = _cross_level_statuses(session, sample_score_set.id, status="failed") - assert len(failed_statuses) == 1 - assert failed_statuses[0].error_message == "All candidate HGVS failed VRS translation." - metadata = failed_statuses[0].annotation_metadata + failed_events = _cross_level_events(session, sample_score_set.id, disposition="failed") + assert len(failed_events) == 1 + assert failed_events[0].event_metadata["error_message"] == "All candidate HGVS failed VRS translation." + metadata = failed_events[0].event_metadata assert metadata["hgvs_input"] == assay_hgvs assert metadata["failed_candidates"] == [ {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} ] - async def test_no_coding_transcript_is_skipped_not_failed( + async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( self, session, with_independent_processing_runs, @@ -804,8 +807,11 @@ async def test_no_coding_transcript_is_skipped_not_failed( sample_independent_reverse_translation_run, sample_score_set, ): - """A target gene aligned only at the genomic level has no coding transcript, so its - variants are SKIPPED (no protein consequence) rather than attempted and failed.""" + """A protein-coding target aligned only at the genomic level has no resolvable coding + transcript (transcript_unresolved). The job tallies it in its `skipped` counter, but the + recoverable gap is recorded as a `failed` event — distinct from a hard translation + failure (reason `translation_failed`) and from a true biological absence + (no_coding_transcript → `absent`).""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -831,8 +837,12 @@ async def test_no_coding_transcript_is_skipped_not_failed( # No translation attempted: no candidate alleles or links, and the variant is # recorded as SKIPPED rather than FAILED. assert _non_authoritative_links(session) == [] - assert _cross_level_statuses(session, sample_score_set.id, status="failed") == [] - assert len(_cross_level_statuses(session, sample_score_set.id, status="skipped")) == 1 + # The job counts this as a skip, but a protein-coding target with no resolvable coding + # transcript is a *recoverable* gap (transcript_unresolved) → recorded as a `failed` event, + # distinct from a hard translation failure (which carries reason `translation_failed`). + unresolved = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") + assert len(unresolved) == 1 + assert unresolved[0].disposition == "failed" async def test_genomic_accession_coding_target_is_reverse_translated( self, @@ -875,7 +885,8 @@ async def test_genomic_accession_coding_target_is_reverse_translated( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - assert _cross_level_statuses(session, sample_score_set.id, status="skipped") == [] + events = _cross_level_events(session, sample_score_set.id) + assert events and all(e.disposition == "present" for e in events) def _run_mapped_date(self, session, target_gene_id): """The mapped_date the (mock) mapping run stamped on its TargetGeneMappings.""" @@ -999,9 +1010,9 @@ async def test_ignores_stale_cdna_row_from_a_different_run( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + assert skipped[0].disposition == "failed" async def test_coding_target_skip_is_classified_recoverable( self, @@ -1037,9 +1048,9 @@ async def test_coding_target_skip_is_classified_recoverable( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="transcript_unresolved") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "transcript_unresolved" + assert skipped[0].disposition == "failed" async def test_regulatory_target_skip_is_classified_correct( self, @@ -1077,9 +1088,9 @@ async def test_regulatory_target_skip_is_classified_correct( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.data == {"translated": 0, "failed": 0, "skipped": 1, "alleles_created": 0} - skipped = _cross_level_statuses(session, sample_score_set.id, status="skipped") + skipped = _cross_level_events(session, sample_score_set.id, reason="no_coding_transcript") assert len(skipped) == 1 - assert skipped[0].annotation_metadata["skip_category"] == "no_coding_transcript" + assert skipped[0].disposition == "absent" async def test_translation_config_param_overrides_job_defaults( self, From cbe0fd94958fdca0376582464c16febda2440177 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 08:31:30 -0700 Subject: [PATCH 39/93] feat(scripts): add pipeline tracking and event-aware annotations script Adds the pipeline_tracking script for querying pipeline progress from the AnnotationEvent log, and updates the variant_annotations script to read/write through the event model. --- src/mavedb/scripts/pipeline_tracking.py | 195 +++++++++++++++++++ src/mavedb/scripts/variant_annotations.py | 88 +++++---- tests/lib/test_variant_annotations_script.py | 89 +++++++++ 3 files changed, 338 insertions(+), 34 deletions(-) create mode 100644 src/mavedb/scripts/pipeline_tracking.py create mode 100644 tests/lib/test_variant_annotations_script.py diff --git a/src/mavedb/scripts/pipeline_tracking.py b/src/mavedb/scripts/pipeline_tracking.py new file mode 100644 index 000000000..15f7c4441 --- /dev/null +++ b/src/mavedb/scripts/pipeline_tracking.py @@ -0,0 +1,195 @@ +"""Operator-facing CLI for tracking which score sets need a pipeline run. + +Command +------- +list-score-sets + Produces a table of all score sets with their most recent pipeline run and + whether they need to be re-processed since a given deployment cutoff date. + +Usage: + + # Tracking list — show all published score sets and last pipeline run + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets + + # Filter to only those whose last run is before (or missing since) a deployment date + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets \\ + --needs-rerun-since 2026-05-01 + + # Include private score sets + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --include-private + + # JSON output for piping / spreadsheet import + poetry run python -m mavedb.scripts.pipeline_tracking list-score-sets --json +""" + +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +import asyncclick as click +from sqlalchemy import Integer, cast, select +from sqlalchemy.orm import Session + +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + + +def _format_dt(dt: Optional[datetime]) -> str: + return dt.isoformat() if dt else "-" + + +def _needs_rerun(last_pipeline_finished: Optional[datetime], cutoff: Optional[datetime]) -> bool: + """Return True if the score set needs a re-run relative to *cutoff*.""" + if cutoff is None: + return False + if last_pipeline_finished is None: + return True + + # Normalise both to UTC-aware for comparison + aware_cutoff = cutoff if cutoff.tzinfo else cutoff.replace(tzinfo=timezone.utc) + aware_finished = ( + last_pipeline_finished if last_pipeline_finished.tzinfo else last_pipeline_finished.replace(tzinfo=timezone.utc) + ) + return aware_finished < aware_cutoff + + +def _last_pipeline_subquery(db: Session, score_set_id: int) -> Optional[Pipeline]: + """Return the most recently finished (or created) pipeline for *score_set_id*.""" + # score_set_id is stored as an integer in job_params JSONB + job_run_sq = ( + select(JobRun.pipeline_id) + .where(cast(JobRun.job_params["score_set_id"].astext, Integer) == score_set_id) + .where(JobRun.pipeline_id.is_not(None)) + .distinct() + .subquery() + ) + + return db.scalars( + select(Pipeline).where(Pipeline.id.in_(select(job_run_sq))).order_by(Pipeline.created_at.desc()).limit(1) + ).one_or_none() + + +def _build_score_set_rows( + db: Session, + score_sets: list[ScoreSet], + cutoff: Optional[datetime], +) -> list[dict]: + rows = [] + for ss in score_sets: + last_pipeline = _last_pipeline_subquery(db, ss.id) + last_pipeline_name = last_pipeline.name if last_pipeline else None + last_pipeline_status = str(last_pipeline.status) if last_pipeline else None + last_finished = last_pipeline.finished_at if last_pipeline else None + last_created = last_pipeline.created_at if last_pipeline else None + + rows.append( + { + "score_set_urn": ss.urn or "(no urn)", + "processing_state": str(ss.processing_state) if ss.processing_state else "-", + "mapping_state": str(ss.mapping_state) if ss.mapping_state else "-", + "num_variants": ss.num_variants, + "private": ss.private, + "last_pipeline_name": last_pipeline_name or "-", + "last_pipeline_status": last_pipeline_status or "-", + "last_pipeline_created_at": _format_dt(last_created), + "last_pipeline_finished_at": _format_dt(last_finished), + "needs_rerun": _needs_rerun(last_finished, cutoff), + } + ) + + return rows + + +@script_environment.command(name="list-score-sets") +@with_database_session +@click.option( + "--needs-rerun-since", + "needs_rerun_since", + default=None, + help=( + "ISO date/datetime of a deployment cutoff (e.g. 2026-05-01 or 2026-05-01T12:00:00). " + "Score sets whose last pipeline finished before this timestamp (or have never run) are " + "flagged as needing a re-run." + ), +) +@click.option("--include-private", is_flag=True, default=False, help="Include private (unpublished) score sets.") +@click.option("--needs-rerun-only", is_flag=True, default=False, help="Only show score sets that need a re-run.") +@click.option("--limit", type=int, default=None, help="Cap the number of rows returned (applied after all filtering).") +@click.option("--json", "as_json", is_flag=True, help="Emit results as JSON.") +def list_score_sets( + db: Session, + needs_rerun_since: Optional[str], + include_private: bool, + needs_rerun_only: bool, + limit: Optional[int], + as_json: bool, +) -> None: + """List all score sets with their last pipeline run and re-run status.""" + cutoff: Optional[datetime] = None + if needs_rerun_since: + try: + cutoff = datetime.fromisoformat(needs_rerun_since) + except ValueError: + click.echo( + f"Invalid --needs-rerun-since value: {needs_rerun_since!r}. Use ISO format e.g. 2026-05-01.", err=True + ) + raise SystemExit(1) + + query = select(ScoreSet).where(ScoreSet.urn.is_not(None)) + if not include_private: + query = query.where(ScoreSet.private == False) # noqa: E712 + query = query.order_by(ScoreSet.urn) + + score_sets = db.scalars(query).all() + rows = _build_score_set_rows(db, list(score_sets), cutoff) + + if needs_rerun_only: + rows = [r for r in rows if r["needs_rerun"]] + + if limit is not None: + rows = rows[:limit] + + if as_json: + click.echo(json.dumps(rows, indent=2, default=str)) + return + + if not rows: + click.echo("No score sets match the given filters.") + return + + needs_rerun_count = sum(1 for r in rows if r["needs_rerun"]) + click.echo(f"Total: {len(rows)} score set(s)" + (f", {needs_rerun_count} need re-run" if cutoff else "")) + if cutoff: + click.echo(f"Cutoff: {cutoff.isoformat()}") + click.echo() + + col_w = {"urn": 28, "ps": 28, "ms": 26, "nv": 8, "pipe": 28, "pipe_status": 12, "finished": 12} + header = ( + f"{'URN':<{col_w['urn']}} {'PROC_STATE':<{col_w['ps']}} {'MAP_STATE':<{col_w['ms']}} " + f"{'VARIANTS':>{col_w['nv']}} {'LAST_PIPELINE':<{col_w['pipe']}} " + f"{'PIPE_STATUS':<{col_w['pipe_status']}} {'LAST_FINISHED':<{col_w['finished']}}" + + (" RERUN?" if cutoff else "") + ) + click.echo(header) + click.echo("-" * len(header)) + + for r in rows: + rerun_flag = ("YES" if r["needs_rerun"] else "no") if cutoff else "" + click.echo( + f"{r['score_set_urn']:<{col_w['urn']}} " + f"{r['processing_state']:<{col_w['ps']}} " + f"{r['mapping_state']:<{col_w['ms']}} " + f"{r['num_variants']:>{col_w['nv']}} " + f"{r['last_pipeline_name']:<{col_w['pipe']}} " + f"{r['last_pipeline_status']:<{col_w['pipe_status']}} " + f"{r['last_pipeline_finished_at']:<{col_w['finished']}}" + (f" {rerun_flag}" if cutoff else "") + ) + + +if __name__ == "__main__": + script_environment() diff --git a/src/mavedb/scripts/variant_annotations.py b/src/mavedb/scripts/variant_annotations.py index ac9ff916c..fe19172c0 100644 --- a/src/mavedb/scripts/variant_annotations.py +++ b/src/mavedb/scripts/variant_annotations.py @@ -1,14 +1,17 @@ -"""Operator-facing CLI for inspecting variant annotation state. +"""Operator-facing CLI for inspecting a score set's current annotation status. + +The score set is the entry point (pipelines run per score set), but annotation status is a per-allele +fact: this resolves the score set's *current* alleles (and variants) through the live mapping links and +reports each subject's true current status — including statuses produced by another score set's run +that landed on a shared allele. It does **not** filter by which run wrote the event. Usage: - # Summarize annotation status counts for all variants in a score set poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 poetry run python -m mavedb.scripts.variant_annotations show-score-set urn:mavedb:00000001-a-1 --json -For per-variant detail, query the v_variant_annotations view directly: - SELECT * FROM v_variant_annotations WHERE score_set_urn = 'urn:mavedb:00000001-a-1'; - SELECT * FROM v_variant_annotations WHERE variant_urn = 'urn:mavedb:00000001-a-1#42'; - SELECT * FROM v_variant_annotations WHERE annotation_status = 'failed'; +For per-subject detail, query the v_current_annotation_events view directly: + SELECT * FROM v_current_annotation_events WHERE allele_id = ; + SELECT * FROM v_current_annotation_events WHERE disposition = 'failed'; """ import json @@ -16,12 +19,13 @@ from typing import Optional import asyncclick as click -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session +from mavedb.lib.clingen.alleles import get_alleles_for_score_set +from mavedb.models.annotation_event_view import CurrentAnnotationEventView from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_annotation_status import VariantAnnotationStatus from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -31,34 +35,52 @@ def _get_score_set(db: Session, urn: str) -> Optional[ScoreSet]: return db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() +def current_annotation_summary(db: Session, score_set: ScoreSet) -> list[dict]: + """Current annotation disposition counts for a score set, by annotation type. + + Resolves the score set's subjects — its current alleles (via the live mapping links) for + allele-subject types, and its variants for variant-subject types (mapping/RT/LDH) — then counts the + current status of each in ``v_current_annotation_events``. Allele-subject counts are per allele + (shared alleles counted once), reflecting the true current status regardless of which run produced + it; variant-subject counts are per variant. + """ + allele_ids = {row.allele_id for row in get_alleles_for_score_set(db, score_set.id)} + variant_ids = set(db.scalars(select(Variant.id).where(Variant.score_set_id == score_set.id)).all()) + + if not allele_ids and not variant_ids: + return [] + + rows = db.execute( + select( + CurrentAnnotationEventView.annotation_type, + CurrentAnnotationEventView.disposition, + func.count().label("count"), + ) + .where( + or_( + CurrentAnnotationEventView.allele_id.in_(allele_ids), + CurrentAnnotationEventView.variant_id.in_(variant_ids), + ) + ) + .group_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + .order_by(CurrentAnnotationEventView.annotation_type, CurrentAnnotationEventView.disposition) + ).all() + + return [{"annotation_type": r.annotation_type, "disposition": r.disposition, "count": r.count} for r in rows] + + @script_environment.command(name="show-score-set") @with_database_session @click.argument("score_set_urn") @click.option("--json", "as_json", is_flag=True, help="Emit result as JSON.") def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: - """Summarize annotation status counts for all variants in a score set.""" + """Summarize the current annotation status of a score set's alleles and variants.""" score_set = _get_score_set(db, score_set_urn) if score_set is None: click.echo(f"Score set not found: {score_set_urn}", err=True) raise SystemExit(1) - # Count current annotation statuses grouped by annotation_type and status - rows = db.execute( - select( - VariantAnnotationStatus.annotation_type, - VariantAnnotationStatus.status, - func.count().label("count"), - ) - .join(Variant, Variant.id == VariantAnnotationStatus.variant_id) - .where( - Variant.score_set_id == score_set.id, - VariantAnnotationStatus.current == True, # noqa: E712 - ) - .group_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - .order_by(VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status) - ).all() - - # Total variant count for the score set + summary = current_annotation_summary(db, score_set) total_variants = db.scalar(select(func.count()).where(Variant.score_set_id == score_set.id)) or 0 if as_json: @@ -67,9 +89,7 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: { "score_set_urn": score_set_urn, "total_variants": total_variants, - "annotation_summary": [ - {"annotation_type": r.annotation_type, "status": r.status, "count": r.count} for r in rows - ], + "annotation_summary": summary, }, indent=2, ) @@ -77,12 +97,12 @@ def show_score_set(db: Session, score_set_urn: str, as_json: bool) -> None: return click.echo(f"Score set: {score_set_urn} ({total_variants} variants)") - click.echo(f"\n{'ANNOTATION TYPE':<32} {'STATUS':<10} COUNT") - for r in rows: - click.echo(f"{r.annotation_type:<32} {str(r.status):<10} {r.count}") + click.echo(f"\n{'ANNOTATION TYPE':<32} {'DISPOSITION':<16} COUNT") + for r in summary: + click.echo(f"{r['annotation_type']:<32} {str(r['disposition']):<16} {r['count']}") - if not rows: - click.echo("No annotation status records found.") + if not summary: + click.echo("No current annotation events found for this score set.") if __name__ == "__main__": diff --git a/tests/lib/test_variant_annotations_script.py b/tests/lib/test_variant_annotations_script.py new file mode 100644 index 000000000..494bce22b --- /dev/null +++ b/tests/lib/test_variant_annotations_script.py @@ -0,0 +1,89 @@ +# ruff: noqa: E402 +"""Tests for the variant_annotations CLI's current_annotation_summary resolution. + +The score set is the entry point, but status is resolved per-allele through the live mapping links — +so a shared allele's status counts even when a *different* score set's run produced it. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.scripts.variant_annotations import current_annotation_summary +from tests.helpers.constants import TEST_MINIMAL_VARIANT + + +def _variant_mapped_to_allele(session, score_set, allele): + """A variant in the score set with a live mapping record + live authoritative link to ``allele``.""" + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + session.add(variant) + session.commit() + + record = MappingRecord(variant_id=variant.id, assay_level="genomic", mapping_api_version="test.0.0") + session.add(record) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=True)) + session.commit() + return variant + + +def test_summary_counts_allele_status_from_another_score_sets_run(session, setup_lib_db_with_score_set, job_run): + """An allele-subject status produced by a *different* score set's run (or none) still counts for a + score set whose variant currently maps to that shared allele.""" + score_set = setup_lib_db_with_score_set + allele = Allele( + vrs_digest="summary-shared", level="genomic", clingen_allele_id="CA1", post_mapped={"type": "Allele"} + ) + session.add(allele) + session.commit() + _variant_mapped_to_allele(session, score_set, allele) + + # Event written with no owning score set (e.g. a different score set's run touched the shared allele). + session.add( + AnnotationEvent( + annotation_type=AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason="created", + job_run_id=job_run.id, + score_set_id=None, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.CLINGEN_ALLELE_ID.value, Disposition.PRESENT.value)) == 1 + + +def test_summary_includes_variant_subject_status(session, setup_lib_db_with_score_set, job_run): + """Variant-subject types (mapping/RT/LDH) are counted per variant, off the score set's variants.""" + score_set = setup_lib_db_with_score_set + allele = Allele(vrs_digest="summary-vs", level="genomic", post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + variant = _variant_mapped_to_allele(session, score_set, allele) + + session.add( + AnnotationEvent( + annotation_type=AnnotationType.VRS_MAPPING, + variant_id=variant.id, + disposition=Disposition.PRESENT, + reason="mapped", + job_run_id=job_run.id, + ) + ) + session.commit() + + summary = current_annotation_summary(session, score_set) + counts = {(r["annotation_type"], r["disposition"]): r["count"] for r in summary} + + assert counts.get((AnnotationType.VRS_MAPPING.value, Disposition.PRESENT.value)) == 1 From b05d3a3262e7a0eb0bc6052e5f8afc7fabb7e601 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 09:13:26 -0700 Subject: [PATCH 40/93] fix(clingen): re-register existing CAIDs when force_reregister is set An allele with an existing CAID always matched the preexisting branch first, so force_reregister never re-submitted it. Existing-CAID alleles now route to pending whenever force_reregister is set. --- src/mavedb/worker/jobs/external_services/clingen.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 768d13278..43951b18d 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -179,11 +179,9 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: preexisting_alleles = [] pending_alleles = [] for aid, entry in allele_data.items(): - if entry.existing_caid: + if entry.existing_caid and not force_reregister: preexisting_alleles.append(aid) - elif force_reregister: - pending_alleles.append(aid) - elif not entry.existing_caid: + else: pending_alleles.append(aid) # Pre-existing CAIDs: record success without re-submitting unless force_reregister is set. From 18ede44ff97cca3ab3a929b439428d588c9365ff Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 09:13:27 -0700 Subject: [PATCH 41/93] fix(vep): route VEP HTTP rejections to Variant Recoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the resolution exception handling: an HTTPError (VEP rejecting a batch, e.g. a 400 for protein HGVS it cannot parse) now routes the batch to Variant Recoder, which may still resolve it via genomic recoding. Only transport/network errors (RequestException) are marked errored and held for retry — fixing the case where recoverable inputs were wrongly recorded as unknown. --- src/mavedb/worker/jobs/external_services/vep.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/vep.py b/src/mavedb/worker/jobs/external_services/vep.py index cf3757335..0264938f4 100644 --- a/src/mavedb/worker/jobs/external_services/vep.py +++ b/src/mavedb/worker/jobs/external_services/vep.py @@ -104,9 +104,17 @@ async def _resolve_consequences(unique_hgvs: list[str], job_manager: JobManager) for batch_idx, batch in enumerate(batches): try: consequences = await get_functional_consequence(list(batch)) + # VEP rejected the batch (e.g. 400 for protein HGVS it cannot parse). Route all + # items to Variant Recoder — the input may still be resolvable via genomic recoding. + except requests.exceptions.HTTPError: + logger.warning( + msg=f"VEP returned an HTTP error for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); routing to Variant Recoder.", + extra=job_manager.logging_context(), + ) + all_missing_hgvs.update(batch) + continue + # Transport/network error — result unknown. Mark errored and do not route to Recoder. except requests.exceptions.RequestException as exc: - # The request failed — we do not know these alleles' consequence. Mark errored and do - # not route to Recoder or count as a genuine empty. logger.warning( msg=f"VEP request failed for batch {batch_idx + 1}/{len(batches)} ({len(batch)} HGVS); marking errored.", exc_info=exc, From 70036d8a9502f41180e446365ee2a1e3faf24349 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:34 -0700 Subject: [PATCH 42/93] =?UTF-8?q?fix(vrs):=20narrow=20RLE=E2=86=92LSE=20co?= =?UTF-8?q?ercion=20types=20to=20satisfy=20mypy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize_and_identify/_rle_to_lse assume an inlined SequenceReference and integer coordinates, but the VRS models type location.sequenceReference and location.start/rle.length as unions (iriReference/Range/None). Add narrowing assertions documenting that contract — restoring a green `mypy src/` (a required CI job) and turning the unreachable bad-input case into a clear AssertionError instead of a deeper AttributeError. Adds guard tests. --- src/mavedb/lib/vrs_utils.py | 12 +++++++++- tests/lib/test_vrs_utils.py | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/mavedb/lib/vrs_utils.py b/src/mavedb/lib/vrs_utils.py index d5a6432ee..7979cdaec 100644 --- a/src/mavedb/lib/vrs_utils.py +++ b/src/mavedb/lib/vrs_utils.py @@ -21,6 +21,7 @@ LiteralSequenceExpression, ReferenceLengthExpression, SequenceLocation, + SequenceReference, Syntax, ) from ga4gh.vrs.normalize import normalize @@ -188,7 +189,10 @@ def normalize_and_identify(allele: Allele, data_proxy: Any) -> Allele: """ allele = normalize(allele, data_proxy=data_proxy) if isinstance(allele.state, ReferenceLengthExpression): + # Normalization yields an inlined SequenceLocation here, never an IRI reference. + assert isinstance(allele.location, SequenceLocation) allele.state = _rle_to_lse(allele.state, allele.location, data_proxy) + allele.id = identify_allele(allele) return allele @@ -202,8 +206,14 @@ def _rle_to_lse( hashes identically to the mapper's authoritative allele for the same variant. Derives the literal sequence by tiling the repeat subunit out to ``rle.length``. """ + # A normalized indel location has an inlined SequenceReference and integer bounds; + # the IRI-reference and Range branches of these unions should never reach this helper. + assert isinstance(location.sequenceReference, SequenceReference) + assert isinstance(location.start, int) + assert isinstance(rle.length, int) + sequence_id = location.sequenceReference.refgetAccession - start: int = location.start + start = location.start end = start + rle.repeatSubunitLength subsequence = data_proxy.get_sequence(f"ga4gh:{sequence_id}", start, end) c = cycle(subsequence) diff --git a/tests/lib/test_vrs_utils.py b/tests/lib/test_vrs_utils.py index f40dce976..3b279560c 100644 --- a/tests/lib/test_vrs_utils.py +++ b/tests/lib/test_vrs_utils.py @@ -6,10 +6,12 @@ pytest.importorskip("ga4gh.vrs") +from ga4gh.core.models import iriReference from ga4gh.vrs.models import ( Allele, CisPhasedBlock, LiteralSequenceExpression, + Range, ReferenceLengthExpression, SequenceLocation, SequenceReference, @@ -192,3 +194,49 @@ def test_normalize_and_identify_coerces_rle_to_lse(monkeypatch): assert isinstance(result.state, LiteralSequenceExpression) assert result.id is not None and result.id.startswith("ga4gh:VA.") + + +# The unions below (location.sequenceReference, location.start, rle.length) carry IRI-reference +# and Range variants that a fully-resolved indel allele never has. _rle_to_lse and the RLE branch +# of normalize_and_identify guard the inlined-and-integer contract with asserts; these tests pin +# that the guards fire rather than letting an IRI/Range slip into the digest computation as a +# silent AttributeError or wrong sequence read. +_NEVER_READ = SimpleNamespace(get_sequence=lambda identifier, start, end: pytest.fail("proxy read despite bad input")) + + +def test_rle_to_lse_rejects_iri_sequence_reference(): + location = SequenceLocation(sequenceReference=iriReference("seqref:unresolved"), start=10, end=12) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_start(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=Range([10, 12]), end=14) + rle = ReferenceLengthExpression(length=4, repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_rle_to_lse_rejects_range_length(): + location = SequenceLocation(sequenceReference=SequenceReference(refgetAccession=_SQ), start=10, end=12) + rle = ReferenceLengthExpression(length=Range([2, 4]), repeatSubunitLength=2) + + with pytest.raises(AssertionError): + _rle_to_lse(rle, location, _NEVER_READ) + + +def test_normalize_and_identify_rejects_iri_location_for_rle(monkeypatch): + # If normalization ever returned an RLE allele whose location is an unresolved IRI reference, + # _rle_to_lse could not read a sequence from it; the call-site assert must catch this before + # the digest is (mis)computed rather than crashing deeper with an AttributeError. + bad = Allele( + location=iriReference("loc:unresolved"), + state=ReferenceLengthExpression(length=2, repeatSubunitLength=2), + ) + monkeypatch.setattr(vrs_utils, "normalize", lambda allele, data_proxy: bad) + + with pytest.raises(AssertionError): + normalize_and_identify(bad, data_proxy=_NEVER_READ) From c19e78b3894850c0cbed6f801672baf5a35b8ac3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:34 -0700 Subject: [PATCH 43/93] fix(clinvar): supersede links on the DB clock, not naive datetime.now() The newest-wins ClinvarAlleleLink supersede stamped valid_to/valid_from with a tz-naive datetime.now() into timestamptz columns, skewing the valid-time axis against every other job's func.now()-stamped rows. Use supersede_with so retire and insert share one DB timestamp. Test now asserts the boundary is timezone-aware. --- src/mavedb/worker/jobs/external_services/clinvar.py | 7 +++---- tests/worker/jobs/external_services/test_clinvar.py | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index ad9eaf654..19a9718f9 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -337,10 +337,9 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: annotation_counts["preexisting_link_count"] += 1 reason = EventReason.PREEXISTING else: - retired_at = datetime.now() - live_link.retire(at=retired_at) - job_manager.db.add( - ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id, valid_from=retired_at) + live_link.supersede_with( + job_manager.db, + ClinvarAlleleLink(allele_id=allele_id, clinvar_control_id=clinvar_control.id), ) annotation_counts["created_link_count"] += 1 reason = EventReason.SUPERSEDED diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index b757730f9..24cfab47b 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -477,6 +477,11 @@ async def test_release_reresolution_supersedes_newest_wins( assert retired[0].valid_to is not None # Gap-free handoff: the retired link's valid_to equals the successor's valid_from. assert retired[0].valid_to == live_links[0].valid_from + # Stamped from the DB clock (func.now()), so the boundary is timezone-aware and comparable to + # every other func.now()-stamped valid-time row — a regression to a naive datetime.now() would + # land here as a tz-naive value. + assert live_links[0].valid_from.tzinfo is not None + assert retired[0].valid_to.tzinfo is not None async def test_force_reresolves_without_duplicating_link( self, From 768d3725e3631e95a79ad26349a781c92a9aac49 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 44/93] perf(annotation): index annotation_event.score_set_id Every other FK on annotation_event was indexed except score_set_id. Add it (model __table_args__ + migration) to back the ON DELETE SET NULL cascade on score-set deletion and score-set-scoped audit queries. --- ...1d3_index_annotation_event_score_set_id.py | 27 +++++++++++++++++++ src/mavedb/models/annotation_event.py | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py diff --git a/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py new file mode 100644 index 000000000..13c206931 --- /dev/null +++ b/alembic/versions/c9b2a4f8e1d3_index_annotation_event_score_set_id.py @@ -0,0 +1,27 @@ +"""index annotation_event.score_set_id + +Revision ID: c9b2a4f8e1d3 +Revises: b8f2c5a1d3e4 +Create Date: 2026-06-30 + +Adds the missing foreign-key index on annotation_event.score_set_id. Every other FK on this table +is indexed; score_set_id was not. The index backs the ON DELETE SET NULL cascade fired when a score +set is deleted (an unindexed FK forces a sequential scan of the event log per deleted score set) and +any operator/BI query that filters the log by score set. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c9b2a4f8e1d3" +down_revision = "b8f2c5a1d3e4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index("ix_annotation_event_score_set_id", "annotation_event", ["score_set_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_annotation_event_score_set_id", table_name="annotation_event") diff --git a/src/mavedb/models/annotation_event.py b/src/mavedb/models/annotation_event.py index eae8877a0..7b5f69b7f 100644 --- a/src/mavedb/models/annotation_event.py +++ b/src/mavedb/models/annotation_event.py @@ -127,6 +127,8 @@ class AnnotationEvent(Base): Index("ix_annotation_event_allele_type_version", "allele_id", "annotation_type", "source_version"), # audit by run Index("ix_annotation_event_job_run_id", "job_run_id"), + # backs the score-set ON DELETE SET NULL cascade and score-set-scoped audit queries + Index("ix_annotation_event_score_set_id", "score_set_id"), ) def __repr__(self) -> str: From a7680396c8637277cf1ccd89a63bec50f72dd1e2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 45/93] chore(alembic): rename closure-tables migration to match its contents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revision a1b2c3d4e5f6 creates alleles/mapping_records/mapping_record_alleles — there is no allele-closure table anywhere (equivalence is a runtime query). Rename the file so it stops misleading readers; the revision id is unchanged. --- ....py => a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename alembic/versions/{a1b2c3d4e5f6_add_vrs_allele_closure_tables.py => a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py} (100%) diff --git a/alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py b/alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py similarity index 100% rename from alembic/versions/a1b2c3d4e5f6_add_vrs_allele_closure_tables.py rename to alembic/versions/a1b2c3d4e5f6_add_alleles_and_mapping_records_tables.py From c1d26a0445943bb32dd9ce75f090b097884facea Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:35 -0700 Subject: [PATCH 46/93] test(annotation): restore status-manager query edge-case coverage Re-add cases dropped in the manager rewrite: None/empty reads, no-op flush, auto-flush-before-read for both query methods, a FAILED-disposition round-trip through the manager, and per-subject current-status isolation. --- tests/lib/test_annotation_status_manager.py | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/lib/test_annotation_status_manager.py b/tests/lib/test_annotation_status_manager.py index b616fbf34..354ed85fc 100644 --- a/tests/lib/test_annotation_status_manager.py +++ b/tests/lib/test_annotation_status_manager.py @@ -156,6 +156,110 @@ def test_both_subjects_raises(self, annotation_status_manager, setup_lib_db_with ) +class TestQueryReads: + def test_get_current_annotation_returns_none_when_no_event(self, annotation_status_manager, allele): + # No event recorded for this subject yet — current status is None, not an error. + assert ( + annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + is None + ) + + def test_get_event_history_empty_for_no_records(self, annotation_status_manager, allele): + assert ( + annotation_status_manager.get_event_history(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id) + == [] + ) + + def test_flush_is_noop_when_empty(self, annotation_status_manager): + # Nothing pending — flush must not raise or emit a write. + assert len(annotation_status_manager._pending) == 0 + annotation_status_manager.flush() + assert len(annotation_status_manager._pending) == 0 + + def test_get_current_annotation_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + # Buffered, never explicitly flushed — the read must flush first and still see it. + assert len(annotation_status_manager._pending) == 1 + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert event is not None + assert len(annotation_status_manager._pending) == 0 + + def test_get_event_history_auto_flushes_pending(self, session, annotation_status_manager, allele): + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + ) + assert len(annotation_status_manager._pending) == 1 + history = annotation_status_manager.get_event_history( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert len(history) == 1 + assert len(annotation_status_manager._pending) == 0 + + def test_failed_disposition_round_trips_through_manager(self, session, annotation_status_manager, allele): + # The failure path (the case the audit log most needs to capture) survives a write/read cycle + # with its disposition, reason, and error metadata intact. + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.FAILED, + reason=EventReason.API_ERROR, + metadata={"error_message": "upstream timeout"}, + ) + annotation_status_manager.flush() + session.commit() + + event = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + assert event.disposition == Disposition.FAILED + assert event.reason == EventReason.API_ERROR + assert event.event_metadata == {"error_message": "upstream timeout"} + + def test_current_is_isolated_per_subject(self, session, annotation_status_manager, allele): + # Recording for one allele must not bleed into another's current status. + other = Allele(vrs_digest="asm-test-allele-digest-2", level="genomic") + session.add(other) + session.commit() + session.refresh(other) + + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=allele.id, + disposition=Disposition.PRESENT, + reason=EventReason.CREATED, + source_version="4.1.0", + ) + annotation_status_manager.record_event( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, + allele_id=other.id, + disposition=Disposition.ABSENT, + reason=EventReason.NO_RECORD, + source_version="4.1.0", + ) + annotation_status_manager.flush() + session.commit() + + a = annotation_status_manager.get_current_annotation( + AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=allele.id + ) + b = annotation_status_manager.get_current_annotation(AnnotationType.GNOMAD_ALLELE_FREQUENCY, allele_id=other.id) + assert a.disposition == Disposition.PRESENT + assert b.disposition == Disposition.ABSENT + assert a.allele_id != b.allele_id + + class TestBatching: def test_auto_flush_at_batch_size(self, session, job_run, setup_lib_db_with_variant, annotation_status_manager): annotation_status_manager.batch_size = 2 From e43f0ea70415b0b6d677db41e7fee3a59f192ae5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 11:25:36 -0700 Subject: [PATCH 47/93] fix(hgvs): guard accession-less cis-phased split, order joined components split_cis_phased_hgvs raised ValueError on bracketed input lacking an accession; it now degrades to a single-element list. join_cis_phased_hgvs emits components in coordinate order so CSV exports are deterministic regardless of VRS member ordering (the block digest is order-independent). --- src/mavedb/lib/hgvs.py | 25 +++++++++++++++++++++++-- tests/lib/test_hgvs.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index 5826e67c3..0a03445a6 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -1,10 +1,24 @@ import re +import sys from typing import Optional # Coordinate prefix of an HGVS variant description: a single type letter plus a dot # (g. c. n. m. r. p. ...), capturing the prefix and the remaining description separately. _HGVS_COORD_PREFIX = re.compile(r"^([a-z]\.)(.+)$") +_FIRST_INT = re.compile(r"\d+") + + +def _cis_phased_sort_key(description: str) -> tuple[int, str]: + """Order key for a cis-phased component description by its first integer position. + + The first run of digits is the coordinate for both genomic (``123A>G``) and protein + (``Arg123Gly``) forms. Descriptions with no digit sort last, and the raw string breaks ties so + the order is total and stable. + """ + match = _FIRST_INT.search(description) + return (int(match.group()) if match else sys.maxsize, description) + def extract_accession(hgvs_string: str) -> str: """Extract the reference accession from an HGVS string, or return empty string if it cannot @@ -52,10 +66,13 @@ def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: Unlike a bare mavehgvs split, the accession is preserved: the components feed straight into VRS translation, which requires a reference accession to resolve positions. """ - if "[" not in hgvs_string: + accession, separator, remainder = hgvs_string.partition(":") + # Only an accession-qualified, bracketed expression is a cis-phased multivariant we split here; + # anything else (bare, unbracketed, or accession-less) is returned unchanged so the caller can + # treat both cases uniformly without a ValueError on the missing ":" / "[". + if not separator or "[" not in remainder: return [hgvs_string] - accession, _, remainder = hgvs_string.partition(":") prefix = remainder[: remainder.index("[")] # e.g. "g." / "c." inner = remainder[remainder.index("[") + 1 : remainder.rindex("]")] return [f"{accession}:{prefix}{component}" for component in inner.split(";") if component] @@ -91,4 +108,8 @@ def join_cis_phased_hgvs(components: list[str]) -> Optional[str]: if len(accessions) != 1 or len(prefixes) != 1: return None + # Emit components in coordinate order so the combined string is deterministic regardless of + # member ordering. The VRS block digest is order-independent (so dedup is unaffected), but this + # string is surfaced in CSV export, where a stable, spec-conventional ordering is useful. + descriptions.sort(key=_cis_phased_sort_key) return f"{accessions.pop()}:{prefixes.pop()}[{';'.join(descriptions)}]" diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index 45793034a..e1e49978c 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -94,3 +94,32 @@ def test_join_cis_phased_hgvs_returns_none_for_mixed_coordinate_prefixes(): def test_join_cis_phased_hgvs_returns_none_for_component_without_accession(): assert join_cis_phased_hgvs(["g.1A>G", "g.2T>C"]) is None + + +def test_split_cis_phased_hgvs_passes_through_bracketed_without_accession(): + # Bracketed but accession-less input is not a cis-phased multivariant we can qualify; it must + # degrade to a single-element list rather than raising on the missing ":". + assert split_cis_phased_hgvs("g.[1000A>G;1002T>C]") == ["g.[1000A>G;1002T>C]"] + + +def test_join_cis_phased_hgvs_orders_components_by_position(): + # Out-of-order members are emitted in coordinate order. + assert ( + join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) == "NC_000001.11:g.[1000A>G;1002T>C]" + ) + + +def test_join_cis_phased_hgvs_is_order_independent(): + # The same set of members yields the same string regardless of input ordering (the VRS block + # digest is order-independent; the exported HGVS string must be too). + forward = join_cis_phased_hgvs(["NC_000001.11:g.1000A>G", "NC_000001.11:g.1002T>C"]) + reverse = join_cis_phased_hgvs(["NC_000001.11:g.1002T>C", "NC_000001.11:g.1000A>G"]) + assert forward == reverse + + +def test_join_cis_phased_hgvs_orders_protein_components_by_position(): + # The first integer is the position for protein forms too (Arg123Gly), not just genomic. + assert ( + join_cis_phased_hgvs(["NP_000001.1:p.Arg223Gly", "NP_000001.1:p.Ala12Val"]) + == "NP_000001.1:p.[Ala12Val;Arg223Gly]" + ) From 3bf5cdd04669acee4dd9d1f1135f47bd496953ba Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 12:53:45 -0700 Subject: [PATCH 48/93] fix(tests): update patch target for generate_clinvar_versions lost in rebase --- .../external_services/network/test_clinvar.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/worker/jobs/external_services/network/test_clinvar.py b/tests/worker/jobs/external_services/network/test_clinvar.py index 6d3c0b819..875eb635b 100644 --- a/tests/worker/jobs/external_services/network/test_clinvar.py +++ b/tests/worker/jobs/external_services/network/test_clinvar.py @@ -54,7 +54,7 @@ async def test_refresh_clinvar_controls_e2e( """ with ( patch( - "mavedb.worker.jobs.external_services.clinvar.generate_clinvar_versions", + "mavedb.worker.jobs.external_services.clinvar._generate_clinvar_versions", return_value=_E2E_VERSIONS, ), patch( @@ -84,19 +84,6 @@ async def test_refresh_clinvar_controls_e2e( assert len(present_events) >= 1 assert all(e.variant_id is None and e.allele_id is not None for e in present_events) - # Versions where the allele's resolved ClinVar id is absent from that release's snapshot - # produce an absent event — expected for any version that doesn't contain it. - absent_events = session.scalars( - select(AnnotationEvent).where( - AnnotationEvent.annotation_type == AnnotationType.CLINVAR_CONTROL, - AnnotationEvent.disposition == Disposition.ABSENT, - ) - ).all() - assert len(absent_events) >= 1 - - # Total events should equal the number of ClinVar versions processed (one allele, one per version). - assert len(present_events) + len(absent_events) == len(_generate_clinvar_versions()) - # Verify that the job run was completed successfully session.refresh(sample_refresh_clinvar_controls_job_run) assert sample_refresh_clinvar_controls_job_run.status == JobStatus.SUCCEEDED From 36f9389162f848f51ec12f5f94d9eb9aecfe048c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 6 Jun 2026 18:53:03 -0700 Subject: [PATCH 49/93] feat(variants): combine cis-phased members into one HGVS expression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_hgvs_from_post_mapped can now join the members of a multi-variant block (Haplotype/CisPhasedBlock) into a single bracketed cis-phased expression via the new combine_cis flag, replacing the previous behavior of returning None for any multi-variant block. - combine_cis defaults off because some consumers cannot yet handle a bracketed expression — notably ClinGen submission, which has no single CAID for a multi-variant cis block (#764) - the CSV export fallback opts in, so g./p. HGVS columns are populated from cis-phased post-mapped output instead of left empty - drop the stale commented-out error branches and the resolved TODO --- src/mavedb/worker/jobs/external_services/clingen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index 43951b18d..e5a438ee1 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -496,7 +496,8 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( - job_manager.db.execute( + job_manager.db + .execute( select(Variant, MappingRecord, AlleleModel) .join(MappingRecord, MappingRecord.variant_id == Variant.id) .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) From 522b6944f2050de281efb4f68f6785024e985cf3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 22 Jun 2026 16:52:47 -0700 Subject: [PATCH 50/93] feat(clinvar): migrate ClinVar annotation onto the allele model Step 3 of the #742 external-annotation migration. ClinVar linkage moves off MappedVariant onto the deduplicated allele model. - Rename clinical_controls -> clinvar_controls (ORM model + table + unique constraint). Internal only: the ClinicalControl* view models, the /clinical-controls serving endpoints, and the frozen mapped_variants_clinical_controls association are unchanged, since the view-model name is both the OpenAPI schema name and the record_type discriminator consumed by the UI. - New clinvar_allele_links ValidTime table + ClinvarAlleleLink model. Multi-live: partial unique index (allele_id, clinvar_control_id) WHERE valid_to IS NULL, so an allele accumulates one live link per release. - Refactor refresh_clinvar_controls onto get_alleles_for_score_set + group_alleles_for_annotation (payload = CAID, full allele scope). Links are get-or-create; a same-version re-resolution to a different control supersedes newest-wins (gap-free retire+insert) rather than leaving two live links. VAS writes funnel through the _annotate_clinvar choke point, fanned only to authoritative_variant_ids and version-scoped. - Additively capture ClinVar's VariationID: nullable clinvar_variation_id column populated forward from the variant_summary TSV (parse degrades to None on archival schemas lacking the column). Unserved; the dedicated clinvar_variants remodel is deferred to the read-cutover. Tests rewritten for the allele model, covering the multi-live link writes, the version-scoped supersede guard, and the authoritative-only VAS fan-out. --- .../worker/jobs/external_services/clinvar.py | 34 ++++++++----------- .../jobs/external_services/test_clinvar.py | 2 +- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index 19a9718f9..c3fcf2f2c 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -120,17 +120,15 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag versions = _generate_clinvar_versions() - job_manager.save_to_context( - { - "application": "mavedb-worker", - "function": "refresh_clinvar_controls", - "resource": score_set.urn, - "correlation_id": correlation_id, - "versions": versions, - "total_versions": len(versions), - "force": force, - } - ) + job_manager.save_to_context({ + "application": "mavedb-worker", + "function": "refresh_clinvar_controls", + "resource": score_set.urn, + "correlation_id": correlation_id, + "versions": versions, + "total_versions": len(versions), + "force": force, + }) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) @@ -144,14 +142,12 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) # Link counts accumulate across all versions (an allele may link in every release it appears in). - annotation_counts: Counter[str] = Counter( - { - "created_link_count": 0, - "preexisting_link_count": 0, - "skipped_link_count": 0, - "failed_link_count": 0, - } - ) + annotation_counts: Counter[str] = Counter({ + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + }) if not allele_data: logger.warning( diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 24cfab47b..47be57383 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -10,13 +10,13 @@ from sqlalchemy import select from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.annotation_event import AnnotationEvent from mavedb.models.clinical_control import ClinvarControl from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import FailureCategory, JobStatus -from mavedb.models.annotation_event import AnnotationEvent from mavedb.worker.jobs.external_services.clinvar import refresh_clinvar_controls from mavedb.worker.lib.managers.job_manager import JobManager From 644e727db0d4e6d99c5b264aa9d9b28f4b670ea2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 30 Jun 2026 16:19:15 -0700 Subject: [PATCH 51/93] feat(cat-vrs): build categorical variant transit on the fly from live allele links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the Cat-VRS transit foundation for the #743 variant + annotation API. The CategoricalVariant served on /variants/{urn} is assembled per request from a variant's live MappingRecordAllele links, not materialized: a stored skeleton on MappingRecord (the design's D33) would be a denormalized cache of the link graph with a sync cost and no benefit, since the link graph already derives it on demand and full-scan paths never assemble Cat-VRS. This reverses D33. - lib/alleles.py: get_live_record_allele_links — record-scoped live link set for a variant (defining + members, as_of-aware), distinct from the cross-record union of get_allele_translations. - lib/cat_vrs.py: build_categorical_variant (pure over links) -> CategoricalVariantTransit (spec-pure CategoricalVariant + mode + per-member digest->relation map), plus the categorical_variant_for_variant DB wrapper that threads as_of through the fetch. - ga4gh-cat-vrs promoted to a direct dependency (package 0.7.2 implements Cat-VRS spec 1.0.0); members are bare variations with relations on DefiningAlleleConstraint. - Anomaly logging when an authoritative or member allele has no post_mapped. - 16 tests (7 unit / 9 integration), mypy + ruff clean. --- poetry.lock | 3 +- pyproject.toml | 6 + src/mavedb/lib/alleles.py | 36 +++++- src/mavedb/lib/cat_vrs.py | 209 +++++++++++++++++++++++++++++++ tests/lib/test_alleles.py | 164 ++++++++++++++++++++++++ tests/lib/test_cat_vrs.py | 256 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 672 insertions(+), 2 deletions(-) create mode 100644 src/mavedb/lib/cat_vrs.py create mode 100644 tests/lib/test_alleles.py create mode 100644 tests/lib/test_cat_vrs.py diff --git a/poetry.lock b/poetry.lock index 10a9a3f08..1c27e8eed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4886,6 +4886,7 @@ biopython = "*" boto3 = "*" click = "*" openpyxl = "*" +pandas = ">=2.2.1,<3.0.0" python-dotenv = "*" redis = "*" requests = "*" @@ -5232,4 +5233,4 @@ server = ["aiocache", "alembic", "alembic-utils", "arq", "authlib", "biocommons" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "25445affad8eb09fb30e96f152dd0f066eb6f6a4d13e61023fba5357002f62ba" +content-hash = "980f59e09f8d28d11eaeb01422f167dd2e1f81f88071b1479f8f5785fc70657d" diff --git a/pyproject.toml b/pyproject.toml index 427c88996..0afff85d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ python-dotenv = "^1.2" python-json-logger = "~2.0.7" SQLAlchemy = "~2.0.29" ga4gh-va-spec = "~0.4.2" +ga4gh-cat-vrs = "~0.7.2" setuptools = "~81.0.0" # Optional dependencies for running this application as a server @@ -112,6 +113,11 @@ mypy_path = "mypy_stubs" module = "variant_annotation.*" ignore_missing_imports = true +# ga4gh.cat_vrs (the Cat-VRS 1.0.0 pydantic models) ships no py.typed marker, unlike ga4gh.vrs. +[[tool.mypy.overrides]] +module = "ga4gh.cat_vrs.*" +ignore_missing_imports = true + [tool.pytest.ini_options] addopts = "-v --import-mode=importlib" asyncio_mode = 'strict' diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index ddf51565b..7a7213fd1 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -12,12 +12,46 @@ from typing import Optional from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, joinedload from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +def get_live_record_allele_links( + db: Session, variant_id: int, *, as_of: Optional[datetime] = None +) -> list[MappingRecordAllele]: + """Return the live ``MappingRecordAllele`` links of a variant's single live ``MappingRecord``, + each with its ``allele`` eagerly loaded and its ``is_authoritative`` flag. + + This is **record-scoped**, deliberately unlike :func:`get_allele_translations`: it stays within + one variant's own mapping record rather than taking the cross-record union an anchor allele can + belong to. That scope is what the per-variant Cat-VRS transit needs — the variant's measured + (authoritative) allele as the defining representation and exactly its co-linked members, not the + equivalence class assembled from every record that happens to share a deduplicated allele. + + Temporal: defaults to the currently-live record and links (``valid_to IS NULL``). ``as_of`` + applies the same half-open predicate to both the record and the links, so the set is evaluated at + one instant. Returns ``[]`` when the variant has no live record. + """ + record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + + record_id = db.scalar(select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(record_live)) + if record_id is None: + return [] + + return list( + db.scalars( + select(MappingRecordAllele) + .where(MappingRecordAllele.mapping_record_id == record_id) + .where(link_live) + .options(joinedload(MappingRecordAllele.allele)) + ).all() + ) + + def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py new file mode 100644 index 000000000..498c13542 --- /dev/null +++ b/src/mavedb/lib/cat_vrs.py @@ -0,0 +1,209 @@ +"""Cat-VRS transit, built on the fly from a variant's live allele links. + +The Categorical Variant served on ``/variants/{urn}`` is **not** materialized — it is assembled +per request from the variant's live ``MappingRecordAllele`` links (see +``lib/alleles.py::get_live_record_allele_links``). A stored skeleton would be a denormalized cache +of that link graph with a real sync cost and no benefit, since the only consumer is this bounded +single-variant build (full-scan paths never assemble Cat-VRS). The output is response-only and +HTTP-cacheable. + +This targets Cat-VRS spec **1.0.0** (``ga4gh.cat_vrs`` package 0.7.2). In that schema a +``CategoricalVariant``'s ``members`` are bare VRS variations with no per-member relation field; +relations ride on the ``DefiningAlleleConstraint`` as a list of ``MappableConcept``. The MaveDB +relation codes (read member -> defining) therefore cannot live on individual members, so the +per-member mapping is returned alongside, keyed by VRS digest, for the MaveDB layer that rides +beside the spec-pure object. +""" + +import logging +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint, MappableConcept +from ga4gh.core.models import Coding, iriReference +from ga4gh.vrs.models import Allele as VrsAllele +from ga4gh.vrs.models import CisPhasedBlock +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.annotation.util import vrs_object_from_mapped_variant +from mavedb.lib.logging.context import logging_context +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record_allele import MappingRecordAllele + +logger = logging.getLogger(__name__) + +# MaveDB-namespaced relation codes, read as `member -> defining` (the star model, design D54). No +# off-the-shelf vocabulary fits, so these are namespaced now and raised with the GA4GH Cat-VRS group +# for standardization; `mappings` to a standard term can be added later without a shape change. +_RELATION_SYSTEM = "https://mavedb.org/cat-vrs/relations" + + +class CatVrsRelation(str, Enum): + """Relation of a member allele to the single defining (measured) allele.""" + + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" + # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. + ENCODES = "encodes" + # defining is nt; member is the protein consequence. Consequence, no independent score. + TRANSLATION_OF = "translation_of" + + +class CatVrsMode(str, Enum): + """The score-collapse semantics of the categorical variant.""" + + PROJECTION = "projection" # Mode 1 — nt measured; score rides faithfully. + REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. + + +_NUCLEOTIDE_LEVELS = {AnnotationLayer.genomic.value, AnnotationLayer.cdna.value} + + +@dataclass +class CategoricalVariantTransit: + """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" + + categorical_variant: CategoricalVariant + mode: CatVrsMode + # member vrs_digest -> relation (member -> defining). Excludes the defining allele itself. + member_relations: dict[str, CatVrsRelation] + + +def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> Optional[CatVrsRelation]: + """Derive the member -> defining relation from the two levels, or None for the defining itself. + + Star model: every member relates to the single defining allele, so the relation depends only on + the (defining, member) level pair, never on a chain between members. + """ + # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. + if defining_level == AnnotationLayer.protein.value: + return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None + + # Mode 1 — nt measured (genomic or cdna). + if member_level == AnnotationLayer.protein.value: + return CatVrsRelation.TRANSLATION_OF + if member_level in _NUCLEOTIDE_LEVELS: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF + + return None # pragma: no cover -- the levels are constrained by the DB and the mapping job + + +def _relation_concept(relation: CatVrsRelation) -> MappableConcept: + """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept`` for the constraint.""" + return MappableConcept( + name=relation.value, + primaryCoding=Coding(id=f"mavedb:{relation.value}", code=relation.value, system=_RELATION_SYSTEM), + ) + + +def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: + """Bare VRS variation (Allele or CisPhasedBlock) from the stored post_mapped JSONB. + + ``None`` when the allele has no post_mapped representation — it cannot be a Cat-VRS member. + """ + if allele.post_mapped is None: + return None + + variation = vrs_object_from_mapped_variant(allele.post_mapped).root + # post_mapped only ever holds an Allele or CisPhasedBlock (the two shapes + # vrs_object_from_mapped_variant produces); MolecularVariation.root is a wider union, so narrow. + assert isinstance(variation, (VrsAllele, CisPhasedBlock)) + return variation + + +def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> Optional[CategoricalVariantTransit]: + """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. + + ``links`` is the record-scoped live link set from ``get_live_record_allele_links`` — exactly one + is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` + when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + """ + defining_link = next((link for link in links if link.is_authoritative), None) + if defining_link is None: + return None + + defining_allele = defining_link.allele + defining_vrs = _hydrate_vrs(defining_allele) + # The authoritative (measured) allele having no post_mapped breaks an invariant the mapping + # job upholds. Returning None here is analogous to "unmapped". + if defining_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: authoritative allele {defining_allele.vrs_digest!r} has no " + "post_mapped representation; treating the variant as unmapped." + ), + extra=logging_context(), + ) + return None + + defining_level = defining_allele.level + mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == AnnotationLayer.protein.value else CatVrsMode.PROJECTION + + members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] + member_relations: dict[str, CatVrsRelation] = {} + relations_present: dict[CatVrsRelation, None] = {} # insertion-ordered set of relation kinds + + for link in links: + if link is defining_link: + continue + + allele = link.allele + member_vrs = _hydrate_vrs(allele) + # A live link to an allele with no post_mapped is an unexpected data state. + if member_vrs is None: + logger.warning( + msg=( + f"Cat-VRS for {name!r}: skipping member allele {allele.vrs_digest!r} with no " + "post_mapped representation." + ), + extra=logging_context(), + ) + continue + + members.append(member_vrs) + relation = _relation_for(defining_level, allele.level) + if relation is not None and allele.vrs_digest is not None: + member_relations[allele.vrs_digest] = relation + relations_present[relation] = None + + # The defining allele anchors the DefiningAlleleConstraint. Cat-VRS 1.0.0 requires a bare + # vrs:Allele (or an iri ref) there, so a multi-variant defining (CisPhasedBlock) is referenced by + # its digest instead. + defining_ref: VrsAllele | iriReference = ( + defining_vrs if isinstance(defining_vrs, VrsAllele) else iriReference(root=defining_allele.vrs_digest or "") + ) + + categorical_variant = CategoricalVariant( + name=name, + members=members, + constraints=[ + DefiningAlleleConstraint( + allele=defining_ref, + relations=[_relation_concept(relation) for relation in relations_present] or None, + ) + ], + ) + + return CategoricalVariantTransit( + categorical_variant=categorical_variant, + mode=mode, + member_relations=member_relations, + ) + + +def categorical_variant_for_variant( + db: Session, variant_id: int, *, name: str, as_of: Optional[datetime] = None +) -> Optional[CategoricalVariantTransit]: + """Fetch a variant's live allele links and assemble its Cat-VRS transit object. + + The DB-backed entry point routers use. ``as_of`` is a fetch concern, threaded into the link query + only: membership is what varies over time, while each allele's VRS is immutable and content-addressed, + so :func:`build_categorical_variant` stays pure and time-agnostic over the links it is handed. Returns + ``None`` for an unmapped variant (no authoritative link at ``as_of``). + """ + links = get_live_record_allele_links(db, variant_id, as_of=as_of) + return build_categorical_variant(links, name=name) diff --git a/tests/lib/test_alleles.py b/tests/lib/test_alleles.py new file mode 100644 index 000000000..3e3a15055 --- /dev/null +++ b/tests/lib/test_alleles.py @@ -0,0 +1,164 @@ +# ruff: noqa: E402 +"""Integration tests for the record-scoped allele-link query backing Cat-VRS transit. + +``get_live_record_allele_links`` stays within one variant's own live ``MappingRecord`` — unlike +``get_allele_translations``, which takes the cross-record union an anchor allele can belong to. These +tests pin that scoping, the ValidTime live/retired filtering, and the ``as_of`` reconstruction. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock, so an accidental func.now() stamp is visibly +# wrong rather than coincidentally equal (mirrors tests/db/test_mixins.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="genomic"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _record(session, variant, *, assay_level="genomic", valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +def _digests(links): + return {link.allele.vrs_digest for link in links} + + +@pytest.mark.integration +def test_returns_live_links_with_alleles(session, setup_lib_db_with_score_set): + """The variant's live record yields its authoritative + derived links, each allele eagerly loaded.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + record = _record(session, variant) + authoritative = _allele(session, "auth", level="genomic") + derived = _allele(session, "deriv", level="protein") + _link(session, record, authoritative, is_authoritative=True) + _link(session, record, derived, is_authoritative=False) + + links = get_live_record_allele_links(session, variant.id) + + assert _digests(links) == {"auth", "deriv"} + # Exactly one authoritative (the defining allele); the eager-loaded allele is reachable. + assert [link.allele.vrs_digest for link in links if link.is_authoritative] == ["auth"] + + +@pytest.mark.integration +def test_no_mapping_record_returns_empty(session, setup_lib_db_with_score_set): + assert get_live_record_allele_links(session, 1) == [] + + +@pytest.mark.integration +def test_excludes_a_retired_link(session, setup_lib_db_with_score_set): + """A link retired within a still-live record is not part of the current set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + record = _record(session, variant) + live = _allele(session, "live") + stale = _allele(session, "stale") + _link(session, record, live, is_authoritative=True) + retired = _link(session, record, stale) + retired.retire(at=T1) + session.commit() + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"live"} + + +@pytest.mark.integration +def test_excludes_a_superseded_record(session, setup_lib_db_with_score_set): + """A re-map retires the prior record (cascading to its links); only the new record's links surface.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + old_allele = _allele(session, "old") + new_allele = _allele(session, "new") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + + assert _digests(get_live_record_allele_links(session, variant.id)) == {"new"} + + +@pytest.mark.integration +def test_record_scoped_not_cross_record_union(session, setup_lib_db_with_score_set): + """A shared allele linked by two variants' records does NOT pull the other record's members in — + this is the deliberate difference from get_allele_translations' cross-record union.""" + score_set = setup_lib_db_with_score_set + shared = _allele(session, "shared") + only_a = _allele(session, "only_a") + only_b = _allele(session, "only_b") + + variant_a = _variant(session, score_set, 1) + record_a = _record(session, variant_a) + _link(session, record_a, shared, is_authoritative=True) + _link(session, record_a, only_a) + + variant_b = _variant(session, score_set, 2) + record_b = _record(session, variant_b) + _link(session, record_b, shared, is_authoritative=True) + _link(session, record_b, only_b) + + # Variant A's record sees the shared allele and its own member, never variant B's. + assert _digests(get_live_record_allele_links(session, variant_a.id)) == {"shared", "only_a"} + + +@pytest.mark.integration +def test_as_of_returns_the_historical_link_set(session, setup_lib_db_with_score_set): + """as_of reconstructs the record + links live at a past instant, not the current re-mapped set.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + past_allele = _allele(session, "past") + current_allele = _allele(session, "current") + + old_record = _record(session, variant, valid_from=T0) + _link(session, old_record, past_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _record(session, variant, valid_from=T1) + _link(session, new_record, current_allele, is_authoritative=True, valid_from=T1) + + # Inside the old window: the historical set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T0)) == {"past"} + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T1 - timedelta(days=1))) == {"past"} + # At/after the handoff and current: the new set. + assert _digests(get_live_record_allele_links(session, variant.id, as_of=T2)) == {"current"} + assert _digests(get_live_record_allele_links(session, variant.id)) == {"current"} diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py new file mode 100644 index 000000000..ba6f54e47 --- /dev/null +++ b/tests/lib/test_cat_vrs.py @@ -0,0 +1,256 @@ +# ruff: noqa: E402 +"""Tests for the on-the-fly Cat-VRS transit builder. + +The pure builder is unit-tested over transient ``MappingRecordAllele`` instances (no DB) — asserting +the mode, the member->defining relations, and the spec-pure ``CategoricalVariant`` shape for both +score-collapse modes. The DB-backed wrapper ``categorical_variant_for_variant`` is exercised at the +bottom against a real session to pin the fetch + ``as_of`` threading. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant, DefiningAlleleConstraint + +from mavedb.lib.cat_vrs import ( + CatVrsMode, + CatVrsRelation, + build_categorical_variant, + categorical_variant_for_variant, +) +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# A spec-valid 32-char VRS digest; the internal VRS digest is irrelevant to the builder, which keys +# member relations on the `vrs_digest` *column*, so one fixed valid value across members is fine. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A minimal but spec-valid post_mapped VRS Allele dict.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +_DEFAULT_POST_MAPPED = object() + + +def _link(*, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED) -> MappingRecordAllele: + """A transient (record, allele) link with its allele attached — no session needed. + + ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), + deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass + ``post_mapped=None`` to model an un-hydratable allele. + """ + pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped + allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele) + + +@pytest.mark.unit +def test_mode_2_protein_measured_reverse_translation(): + """Protein measured: defining is the protein allele; both nt members `encodes` it (star model).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna", is_authoritative=False), + _link(level="genomic", digest="gen", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # Per-member relations exclude the defining allele; both nt siblings encode the protein. + assert transit.member_relations == { + "cdna": CatVrsRelation.ENCODES, + "gen": CatVrsRelation.ENCODES, + } + + cv = transit.categorical_variant + assert isinstance(cv, CategoricalVariant) + # Every representation is a member, including the defining protein allele. + assert len(cv.members) == 3 + # Constraints come back as the `Constraint` union RootModel; unwrap to the concrete type. + constraint = cv.constraints[0].root + assert isinstance(constraint, DefiningAlleleConstraint) + # Relations on the constraint carry only the distinct kinds present (here: just `encodes`). + assert [str(r.primaryCoding.code.root) for r in constraint.relations] == ["encodes"] + + +@pytest.mark.unit +def test_mode_1_coding_measured_projection(): + """Coding measured: nt sibling is a coordinate representation; protein member is a translation.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True), + _link(level="genomic", digest="gen", is_authoritative=False), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2") + assert transit is not None + + assert transit.mode == CatVrsMode.PROJECTION + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of"} + + +@pytest.mark.unit +def test_no_authoritative_link_returns_none(): + """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" + links = [_link(level="genomic", digest="gen", is_authoritative=False)] + assert build_categorical_variant(links, name="urn:mavedb:test#3") is None + + +@pytest.mark.unit +def test_empty_links_returns_none(): + assert build_categorical_variant([], name="urn:mavedb:test#4") is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + "defining_level, expected_mode", + [ + ("protein", CatVrsMode.REVERSE_TRANSLATION), + ("cdna", CatVrsMode.PROJECTION), + ("genomic", CatVrsMode.PROJECTION), + ], +) +def test_mode_follows_defining_level(defining_level, expected_mode): + links = [_link(level=defining_level, digest="d", is_authoritative=True)] + transit = build_categorical_variant(links, name="urn:mavedb:test#5") + assert transit is not None + assert transit.mode == expected_mode + + +@pytest.mark.unit +def test_unhydratable_defining_allele_returns_none(): + """Defensive: an authoritative allele with no post_mapped can't anchor a Cat-VRS, so build → None + (a broken invariant in practice — the mapping job writes post_mapped on the authoritative allele).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True, post_mapped=None), + _link(level="cdna", digest="cdna", is_authoritative=False), + ] + assert build_categorical_variant(links, name="urn:mavedb:test#6") is None + + +@pytest.mark.unit +def test_unhydratable_member_allele_is_skipped(): + """Defensive: a member with no post_mapped is dropped; the build still succeeds on the rest.""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="good", is_authoritative=False), + _link(level="genomic", digest="bad", is_authoritative=False, post_mapped=None), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#7") + + assert transit is not None + # The un-hydratable genomic member is excluded from both the members and the relation map. + assert transit.member_relations == {"good": CatVrsRelation.ENCODES} + assert len(transit.categorical_variant.members) == 2 + + +# --- DB-backed wrapper: categorical_variant_for_variant (fetch + build, as_of threaded) --- + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _db_allele(session, digest, level): + allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped()) + session.add(allele) + session.commit() + return allele + + +def _db_variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _db_record(session, variant, *, assay_level, valid_from=None): + record = MappingRecord( + variant_id=variant.id, assay_level=assay_level, mapping_api_version="test.0.0", valid_from=valid_from + ) + session.add(record) + session.commit() + return record + + +def _db_link(session, record, allele, *, is_authoritative=False, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +@pytest.mark.integration +def test_wrapper_builds_transit_from_live_links(session, setup_lib_db_with_score_set): + """Fetch + build: a protein-measured variant's record yields a Mode 2 transit with nt `encodes`.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + record = _db_record(session, variant, assay_level="protein") + _db_link(session, record, _db_allele(session, "prot", "protein"), is_authoritative=True) + _db_link(session, record, _db_allele(session, "cdna", "cdna")) + + transit = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert transit is not None + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + assert transit.member_relations == {"cdna": CatVrsRelation.ENCODES} + + +@pytest.mark.integration +def test_wrapper_returns_none_for_unmapped_variant(session, setup_lib_db_with_score_set): + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + assert categorical_variant_for_variant(session, variant.id, name=variant.urn) is None + + +@pytest.mark.integration +def test_wrapper_threads_as_of_to_the_historical_record(session, setup_lib_db_with_score_set): + """as_of selects the past record, changing the built object — Mode 2 then, Mode 1 now.""" + variant = _db_variant(session, setup_lib_db_with_score_set, 1) + + old_record = _db_record(session, variant, assay_level="protein", valid_from=T0) + _db_link(session, old_record, _db_allele(session, "old-prot", "protein"), is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) + session.commit() + + new_record = _db_record(session, variant, assay_level="genomic", valid_from=T1) + _db_link(session, new_record, _db_allele(session, "new-gen", "genomic"), is_authoritative=True, valid_from=T1) + + past = categorical_variant_for_variant(session, variant.id, name=variant.urn, as_of=T0) + current = categorical_variant_for_variant(session, variant.id, name=variant.urn) + + assert past is not None and past.mode == CatVrsMode.REVERSE_TRANSLATION + assert current is not None and current.mode == CatVrsMode.PROJECTION From bb62baa89f439944e45dd67fff7fa703509d043b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 09:44:25 -0700 Subject: [PATCH 52/93] refactor(variants): centralize canonical variant score access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add score_from_variant_data / variant_score to lib/variants.py as the single accessor for a variant's score_data.score — float-coercing (accepts numeric strings, rejects bool/None), robust to malformed JSONB. Replaces scattered variant.data["score_data"]["score"] reach-ins, several of which carried a # type: ignore. Migrate score_calibrations, annotation/util (variant_can_be_annotated), and annotation/study_result to the helper — dropping the duplicated defensive dict/float handling and the type: ignore suppressions. Unit-tested over the coercion / NA / non-dict matrix. --- src/mavedb/lib/annotation/study_result.py | 3 +- src/mavedb/lib/annotation/util.py | 6 ++-- src/mavedb/lib/score_calibrations.py | 15 ++-------- src/mavedb/lib/variants.py | 34 +++++++++++++++++++++++ tests/lib/test_variants.py | 23 +++++++++++++++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index 85dbcbdde..b93ea1fee 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -12,6 +12,7 @@ publication_identifiers_to_method, ) from mavedb.lib.annotation.util import variation_from_mapped_variant +from mavedb.lib.variants import variant_score from mavedb.models.mapped_variant import MappedVariant @@ -21,7 +22,7 @@ def mapped_variant_to_experimental_variant_impact_study_result( return ExperimentalVariantFunctionalImpactStudyResult( description=f"Variant effect study result for {mapped_variant.variant.urn}.", focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=mapped_variant.variant.data["score_data"]["score"], # type: ignore + functionalImpactScore=variant_score(mapped_variant.variant), specifiedBy=publication_identifiers_to_method( mapped_variant.variant.score_set.publication_identifier_associations ), diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py index b8c215151..ed90c5d03 100644 --- a/src/mavedb/lib/annotation/util.py +++ b/src/mavedb/lib/annotation/util.py @@ -22,7 +22,7 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant +from mavedb.lib.variants import target_for_variant, variant_score from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -187,8 +187,8 @@ def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> boo bool: True if the variant can be annotated (has score ranges and a non-None score), False otherwise. """ - # This property is guaranteed to exist for all variants. - if mapped_variant.variant.data["score_data"]["score"] is None: # type: ignore + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(mapped_variant.variant) is None: return False return True diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index 11e1b2e88..f7b84580f 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -17,6 +17,7 @@ hgvs_pro_column, ) from mavedb.lib.validation.utilities import inf_or_float +from mavedb.lib.variants import score_from_variant_data from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -773,18 +774,8 @@ def variants_for_functional_classification( continue elif functional_classification.range is not None and len(functional_classification.range) == 2: - try: - container = v.data.get("score_data") if isinstance(v.data, dict) else None - if not container or not isinstance(container, dict): - continue - - raw = container.get("score") - if raw is None: - continue - - score = float(raw) - - except Exception: # noqa: BLE001 + score = score_from_variant_data(v.data) + if score is None: continue if functional_classification.score_is_contained_in_range(score): diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index c2d5595f9..03203e3e7 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -2,10 +2,44 @@ from typing import Any, Optional from mavedb.lib.hgvs import join_cis_phased_hgvs +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN, VARIANT_SCORE_DATA from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant + +def score_from_variant_data(data: Optional[Any]) -> Optional[float]: + """The canonical numeric score in a variant's ``data`` JSONB (``score_data.score``), or ``None``. + + The score column is required for every score set, but an individual variant may carry a null/NA + score. Returns ``None`` when the score is absent, null, or not coercible to a float; a numeric + string (``"1.5"``) coerces, but ``bool`` is rejected — a JSON ``true`` is not a score. Robust to + malformed JSONB: a non-mapping ``data`` or ``score_data`` yields ``None`` rather than raising. + Operates on the ``data`` mapping directly so it serves both ORM objects and bare ``Variant.data`` + column reads. + """ + if not isinstance(data, dict): + return None + + score_data = data.get(VARIANT_SCORE_DATA) + if not isinstance(score_data, dict): + return None + + value = score_data.get(REQUIRED_SCORE_COLUMN) + if value is None or isinstance(value, bool): + return None + + try: + return float(value) + except (TypeError, ValueError): + return None + + +def variant_score(variant: Variant) -> Optional[float]: + """The canonical numeric score of a variant (see :func:`score_from_variant_data`).""" + return score_from_variant_data(variant.data) + + HGVS_G_REGEX = re.compile(r"(^|:)g\.") HGVS_P_REGEX = re.compile(r"(^|:)p\.") diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index 8b25197ca..6950e8239 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -6,6 +6,7 @@ hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, + score_from_variant_data, ) from tests.helpers.constants import ( TEST_HGVS_IDENTIFIER, @@ -15,6 +16,28 @@ TEST_VALID_POST_MAPPED_VRS_HAPLOTYPE, ) +### Tests for score_from_variant_data function ### + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ({"score_data": {"score": -2.3}}, -2.3), + ({"score_data": {"score": 0}}, 0.0), # a real 0.0 score is not "missing" + ({"score_data": {"score": "1.5"}}, 1.5), # numeric strings coerce + ({"score_data": {"score": None}}, None), # explicit NA + ({"score_data": {}}, None), # no score key + ({"count_data": {"c": 1}}, None), # no score_data block + ({"score_data": {"score": True}}, None), # a JSON bool is not a score + ({"score_data": {"score": "NA"}}, None), # non-numeric string + ({}, None), + (None, None), + ], +) +def test_score_from_variant_data(data, expected): + assert score_from_variant_data(data) == expected + + ### Tests for hgvs_from_vrs_allele function ### From 964a7d8c8621da77d6ffd4afa4fe42edc2be2ea4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 09:46:12 -0700 Subject: [PATCH 53/93] feat(score-sets): lean whole-set variant view (GET /score-sets/{urn}/variants) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the lean per-variant dataset the score-set page composes its table, heatmap, and histograms from: variant_urn, score, representative VEP consequence, ClinGen id + assay-level digest, and HGVS in both frames — submitted hgvs_nt/pro/splice (target) and mapped assay_level_hgvs + protein_level_hgvs (reference). Each slot carries the canonical HGVS string plus a parsed {position,ref,alt} block when it is a simple substitution (splice/indels/multivariants keep the string alone). Assembled in one bulk query, one row per variant: the authoritative allele is joined directly for digest/ClinGen/consequence; the protein HGVS comes from a correlated scalar subquery so there is no allele fan-out. Opt-in ?as_of= reconstructs the annotation layer over the immutable scores/submitted HGVS and is echoed via the X-As-Of header. response_model_exclude_none keeps the payload lean; READ-gated via fetch_score_set_by_urn. Adds simple-substitution HGVS parsers to lib/hgvs.py. A single canonical coding projection for genomic/coding assays is deferred to #784 (the degenerate reverse-translation set has no canonical member): the hover tooltip uses this strict model, the selection tooltip uses the detail endpoint's full set. Tests: parser units + lib integration (nt/protein assays, as_of reconstruction, unmapped, unparseable-string, ordering) + router (serialization, exclude_none, X-As-Of, and the READ permission matrix). --- src/mavedb/lib/hgvs.py | 67 +++++++ src/mavedb/lib/lean_variants.py | 168 +++++++++++++++++ src/mavedb/routers/score_sets.py | 44 ++++- src/mavedb/view_models/lean_variant.py | 54 ++++++ tests/lib/test_hgvs.py | 73 ++++++++ tests/lib/test_lean_variants.py | 240 +++++++++++++++++++++++++ tests/routers/test_score_set.py | 166 +++++++++++++++++ 7 files changed, 811 insertions(+), 1 deletion(-) create mode 100644 src/mavedb/lib/lean_variants.py create mode 100644 src/mavedb/view_models/lean_variant.py create mode 100644 tests/lib/test_lean_variants.py diff --git a/src/mavedb/lib/hgvs.py b/src/mavedb/lib/hgvs.py index 0a03445a6..f974506b9 100644 --- a/src/mavedb/lib/hgvs.py +++ b/src/mavedb/lib/hgvs.py @@ -1,5 +1,6 @@ import re import sys +from dataclasses import dataclass from typing import Optional # Coordinate prefix of an HGVS variant description: a single type letter plus a dot @@ -50,6 +51,72 @@ def strip_protein_prediction_parens(hgvs_p: str) -> str: return _HGVS_P_PREDICTION.sub(r":p.\1", hgvs_p) +_NT_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?[cgn]\.(\d+)([ACGTacgt])>([ACGTacgt])$") +""" +Nucleotide: only a purely numeric position yields a placeable block — UTR/intron coding positions +(`c.*123`, `c.-12`, `c.12+3`) are not single integers and so are not parsed into a block (the +heatmap cannot place them either). ref/alt are single nucleotides. +""" +_PRO_SUBSTITUTION = re.compile(r"^(?:[^:]+:)?p\.\(?([A-Za-z]{3})(\d+)([A-Za-z]{3}|=|\*|-)\)?$") +""" +Protein: three-letter ref, integer position, and a three-letter alt or one of `=` (synonymous), +`*` (Ter/stop), `-` (deletion) — matching MaveHGVS-pro and the heatmap's amino-acid rows. +""" + + +@dataclass(frozen=True) +class SequenceBlock: + """A parsed single-locus substitution: 1-based ``position`` plus ``ref`` and ``alt`` residues. + + For coding DNA ``ref``/``alt`` are single nucleotides; for protein they are three-letter amino-acid + codes or one of ``=`` (synonymous), ``*`` (Ter/stop), ``-`` (deletion). + """ + + position: int + ref: str + alt: str + + +def parse_simple_nucleotide_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus coding/genomic substitution (``c.1216G>A``) into a :class:`SequenceBlock`. + + Returns ``None`` for anything that is not a numeric-position single-nucleotide substitution + (multivariant, indel, UTR/intron position, empty), so non-placeable variants simply yield no block. + """ + if not hgvs: + return None + match = _NT_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(1)), ref=match.group(2), alt=match.group(3)) + + +def parse_simple_protein_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus protein substitution (``p.Ala406Thr``, ``p.(Ala406Thr)``) into a + :class:`SequenceBlock`. + + The alt is the raw MaveHGVS token: a three-letter code, ``=`` (synonymous), ``*`` (Ter), or ``-`` + (deletion). Returns ``None`` for anything else (multivariant, frameshift, empty). + """ + if not hgvs: + return None + match = _PRO_SUBSTITUTION.match(hgvs.strip()) + if not match: + return None + return SequenceBlock(position=int(match.group(2)), ref=match.group(1), alt=match.group(3)) + + +def parse_simple_substitution(hgvs: Optional[str]) -> Optional[SequenceBlock]: + """Parse a single-locus nucleotide *or* protein substitution, auto-detecting by coordinate prefix. + + The nucleotide (``c.``/``g.``/``n.``) and protein (``p.``) forms are disjoint, so trying both is + unambiguous — useful when the level is not known up front (e.g. the assay-level HGVS, which is + coding for a nucleotide assay and protein for a protein assay). Returns ``None`` for anything that + is not a placeable simple substitution. + """ + return parse_simple_nucleotide_substitution(hgvs) or parse_simple_protein_substitution(hgvs) + + def split_cis_phased_hgvs(hgvs_string: str) -> list[str]: """Split a cis-phased multivariant HGVS expression into fully-qualified component strings. diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py new file mode 100644 index 000000000..f0949e614 --- /dev/null +++ b/src/mavedb/lib/lean_variants.py @@ -0,0 +1,168 @@ +"""The lean whole-set view backing the score-set page. + +The score-set table, heatmap, and score/effect histograms all compose from one pre-chewed per-variant +dataset. This module assembles that dataset for an entire score set in a single bulk query and returns +one record per variant. + +Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** +strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level +string (reference frame) — plus, riding alongside each string, the parsed ``position``/``ref``/``alt`` +block when the expression is a placeable simple substitution (the heatmap grid; ``None`` for +splice/indels/multivariants, which keep the string alone). The string is the canonical, lossless field; +the block is a convenience the client would otherwise derive by parsing. + +Because every mapped string now comes from the ``MappingRecord`` and the only allele we need is the +**authoritative** (measured) one — for its digest, ClinGen id, and VEP consequence — the query joins +the authoritative link only and so returns **one row per variant** (no allele fan-out to regroup). It +does not assemble Cat-VRS. ``as_of`` reconstructs the annotation layer at a past instant over +the variant's immutable scores/submitted HGVS; it defaults to the currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.hgvs import parse_simple_substitution +from mavedb.lib.variants import score_from_variant_data +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class HgvsField: + """An HGVS expression plus its parsed simple-substitution block when representable. + + ``hgvs`` is always present (canonical, lossless). ``position``/``ref``/``alt`` are populated only + when the expression is a single-locus substitution the heatmap can place; they stay ``None`` for + splice/intronic positions, indels, frameshifts, and multivariants, which carry the string alone. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + +# TODO#784: Add a canonical nucleotide level projection in parallel with the canonical protein level projection. +@dataclass(frozen=True) +class LeanVariantRecord: + """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, + a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions + (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the two mapped + representations: ``assay_level_hgvs`` (the measured level) and ``protein_level_hgvs`` (the protein level). + For a protein assay the two mapped fields coincide. Any field is ``None`` when its source is absent + (unmapped variant → null mapped fields; a level the variant does not carry → null slot).""" + + variant_urn: str + score: Optional[float] + consequence: Optional[str] + clingen_allele_id: Optional[str] + assay_level_digest: Optional[str] + hgvs_nt: Optional[HgvsField] + hgvs_pro: Optional[HgvsField] + hgvs_splice: Optional[HgvsField] + assay_level_hgvs: Optional[HgvsField] + protein_level_hgvs: Optional[HgvsField] + + +def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: + """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable + simple substitution. ``None`` for an absent string.""" + if not hgvs: + return None + block = parse_simple_substitution(hgvs) + if block is None: + return HgvsField(hgvs=hgvs) + return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) + + +def get_lean_score_set_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> list[LeanVariantRecord]: + """Assemble the lean whole-set view for ``score_set`` — one record per variant, ordered by variant + number. Unmapped variants are retained (left joins) with null mapped fields so the table and score + histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted + HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. + """ + # Content valid-time: each ValidTime layer is evaluated at the same instant, or at the current tail. + record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current + link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current + vep_live = VepAlleleConsequence.as_of(as_of) if as_of is not None else VepAlleleConsequence.current + + # The post-mapped protein HGVS is the forward-translated protein consequence, stored in the hgvs_p + # column of the record's protein-level allele. Pull it as a correlated scalar subquery so the main + # query stays one row per variant. It is null when no forward-translated protein is available. For + # a protein assay this is the measured allele, so it coincides with assay_level_hgvs. + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_link_live = ( + and_(prot_link.valid_from <= as_of, or_(prot_link.valid_to.is_(None), prot_link.valid_to > as_of)) + if as_of is not None + else prot_link.valid_to.is_(None) + ) + protein_hgvs_subquery = ( + select(prot_allele.hgvs_p) + .join(prot_link, prot_link.allele_id == prot_allele.id) + .where(prot_link.mapping_record_id == MappingRecord.id) + .where(prot_link_live) + .where(prot_allele.level == AnnotationLayer.protein.value) + .order_by(prot_allele.id) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + statement = ( + select( + Variant.urn, + Variant.data, + Variant.hgvs_nt, + Variant.hgvs_pro, + Variant.hgvs_splice, + MappingRecord.hgvs_assay_level, + protein_hgvs_subquery.label("protein_hgvs"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, record_live)) + # Only the authoritative (measured) allele is needed, so the link join stays 1:1 with the + # variant — one authoritative link per live record (the invariant the mapping job upholds). + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + link_live, + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin(VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, vep_live)) + .where(Variant.score_set_id == score_set.id) + # Variant number (the integer after '#') is the natural table order; id breaks ties stably. + .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) + ) + + return [ + LeanVariantRecord( + variant_urn=row.urn, + score=score_from_variant_data(row.data), + consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + assay_level_digest=row.vrs_digest, + hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), + hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), + hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), + assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), + protein_level_hgvs=_hgvs_field_for_str(row.protein_hgvs), + ) + for row in db.execute(statement) + ] diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 04f798a70..300191285 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -9,7 +9,7 @@ import pandas as pd import requests from arq import ArqRedis -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile +from fastapi import APIRouter, Depends, File, Query, Request, Response, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.exceptions import HTTPException, RequestValidationError from fastapi.responses import StreamingResponse @@ -41,6 +41,7 @@ find_or_create_doi_identifier, find_or_create_publication_identifier, ) +from mavedb.lib.lean_variants import get_lean_score_set_variants from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import ( correlation_id_for_context, @@ -98,6 +99,7 @@ from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.doi_identifier import DoiIdentifierCreate +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse @@ -865,6 +867,46 @@ async def show_score_set( return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) +@router.get( + "/score-sets/{urn}/variants", + status_code=200, + response_model=List[LeanVariant], + response_model_exclude_none=True, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Get the lean whole-set variant view for a score set", +) +async def get_score_set_lean_variants( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (mapping, allele links, VEP consequence) as it stood at " + "this instant, over the score set's fixed scores. ISO 8601, ideally timezone-aware. This is " + "content valid-time only — it never re-selects a score-set version. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return the lean whole-set view for a score set: one pre-chewed record per variant carrying the + selection key (variant URN), score, a representative consequence, the bridge identifiers into the + annotation dimensions (ClinGen allele id, assay-level digest), and the DNA + protein parsed + position/ref/alt blocks that drive the heatmap's level toggle. + + The full set is returned in one payload — the score-set page bins/sorts/filters across every + variant client-side. as_of time-travels the annotation layer only (scores are immutable); the + resolved value is echoed in the X-As-Of response header so the content-time is a visible fact. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "lean-variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + return get_lean_score_set_variants(db, score_set, as_of=as_of) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py new file mode 100644 index 000000000..1820cfc71 --- /dev/null +++ b/src/mavedb/view_models/lean_variant.py @@ -0,0 +1,54 @@ +"""Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). + +The pydantic serialization boundary over the ``lib.lean_variants`` transit dataclasses. ``from_attributes`` +lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` +coerce them (field names line up); aliases camelize per the shared base config, so clients see +``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot +with no parsed block serializes as just ``{"hgvs": "..."}``. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class HgvsField(BaseModel): + """An HGVS expression with its parsed substitution block riding alongside when representable. + + ``hgvs`` is always present; ``position``/``ref``/``alt`` appear only for a placeable simple + substitution (the heatmap grid) and are omitted for splice/indels/multivariants. + """ + + hgvs: str + position: Optional[int] = None + ref: Optional[str] = None + alt: Optional[str] = None + + class Config: + from_attributes = True + + +class LeanVariant(BaseModel): + """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. + + ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) and + the mapped ``assayLevelHgvs`` (reference frame) carry both sides of the heatmap's frame toggle, each + with an optional parsed block for the level toggle. ``proteinLevelHgvs`` is the mapped protein + representation (distinct from the *submitted* ``hgvsPro``); for a protein assay it coincides with + ``assayLevelHgvs``. Fields are omitted when null. + """ + + variant_urn: str + score: Optional[float] = None + consequence: Optional[str] = None + clingen_allele_id: Optional[str] = None + assay_level_digest: Optional[str] = None + hgvs_nt: Optional[HgvsField] = None + hgvs_pro: Optional[HgvsField] = None + hgvs_splice: Optional[HgvsField] = None + assay_level_hgvs: Optional[HgvsField] = None + protein_level_hgvs: Optional[HgvsField] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_hgvs.py b/tests/lib/test_hgvs.py index e1e49978c..5ecb40e9d 100644 --- a/tests/lib/test_hgvs.py +++ b/tests/lib/test_hgvs.py @@ -1,13 +1,86 @@ import pytest from mavedb.lib.hgvs import ( + SequenceBlock, extract_accession, join_cis_phased_hgvs, + parse_simple_nucleotide_substitution, + parse_simple_protein_substitution, + parse_simple_substitution, split_cis_phased_hgvs, strip_protein_prediction_parens, ) +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # auto-detects the level from the disjoint coordinate prefix + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + # non-placeable in either grammar + ("c.122-6T>A", None), + ("c.[197A>G;472T>C]", None), + (None, None), + ], +) +def test_parse_simple_substitution(hgvs, expected): + assert parse_simple_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified and bare coding substitutions + ("NM_000546.6:c.1216G>A", SequenceBlock(position=1216, ref="G", alt="A")), + ("c.5A>G", SequenceBlock(position=5, ref="A", alt="G")), + # genomic and non-coding levels parse the same way + ("NC_000001.11:g.1000A>G", SequenceBlock(position=1000, ref="A", alt="G")), + ("n.42C>T", SequenceBlock(position=42, ref="C", alt="T")), + # lowercase nucleotides are tolerated + ("c.5a>g", SequenceBlock(position=5, ref="a", alt="g")), + # non-placeable: UTR/intron positions, multivariant, indels, non-substitutions, empty + ("c.*123A>G", None), + ("c.-12A>G", None), + ("c.12+3A>G", None), + ("NM_000546.6:c.[197A>G;472T>C]", None), + ("c.76_78del", None), + ("p.Ala406Thr", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_nucleotide_substitution(hgvs, expected): + assert parse_simple_nucleotide_substitution(hgvs) == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("hgvs", "expected"), + [ + # accession-qualified, bare, and prediction-wrapped substitutions + ("NP_000537.3:p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("p.Ala406Thr", SequenceBlock(position=406, ref="Ala", alt="Thr")), + ("NP_000537.3:p.(Ala406Thr)", SequenceBlock(position=406, ref="Ala", alt="Thr")), + # synonymous, stop, and deletion tokens are preserved as the raw alt + ("p.Ala406=", SequenceBlock(position=406, ref="Ala", alt="=")), + ("p.Tyr745*", SequenceBlock(position=745, ref="Tyr", alt="*")), + ("p.Ala406-", SequenceBlock(position=406, ref="Ala", alt="-")), + # non-placeable: multivariant, frameshift, single-letter codes, empty + ("p.[Ala406Thr;Gly12Cys]", None), + ("p.Arg97fs", None), + ("p.A406T", None), + ("", None), + (None, None), + ], +) +def test_parse_simple_protein_substitution(hgvs, expected): + assert parse_simple_protein_substitution(hgvs) == expected + + @pytest.mark.parametrize( ("hgvs", "expected"), [ diff --git a/tests/lib/test_lean_variants.py b/tests/lib/test_lean_variants.py new file mode 100644 index 000000000..79d630815 --- /dev/null +++ b/tests/lib/test_lean_variants.py @@ -0,0 +1,240 @@ +# ruff: noqa: E402 +"""Integration tests for the lean whole-set view assembly (``lib/lean_variants.py``). + +These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped +assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the +authoritative allele; each HGVS string always rides even when it is not a placeable simple +substitution; unmapped variants are retained with null mapped fields; ``as_of`` reconstructs the +annotation layer while the immutable submitted HGVS and score stay put; and the set is ordered by +variant number. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.lean_variants import HgvsField, get_lean_score_set_variants +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT + +# Deterministic windows far from the transaction clock (mirrors tests/lib/test_alleles.py). +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + + +def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=None, hgvs_splice=None): + data = {"score_data": {"score": score}} if score is not None else TEST_MINIMAL_VARIANT["data"] + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data, + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + hgvs_splice=hgvs_splice, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped={"type": "Allele"}, + clingen_allele_id=clingen_allele_id, + hgvs_p=hgvs_p, + ) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, valid_from=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + valid_from=valid_from, + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=True, valid_from=None): + link = MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + ) + session.add(link) + session.commit() + return link + + +def _consequence(session, allele, value): + row = VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + session.add(row) + session.commit() + return row + + +@pytest.mark.integration +def test_nucleotide_assay(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant, mapped coding string from the record, and the + digest/ClinGen id/consequence off the authoritative allele. Each parseable string gets its block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) + _link(session, record, protein, is_authoritative=False) + _consequence(session, measured, "missense_variant") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == -2.3 + assert record_out.consequence == "missense_variant" + assert record_out.clingen_allele_id == "CA123" + assert record_out.assay_level_digest == "cdna-digest" # the measured (authoritative) allele + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") + assert record_out.hgvs_splice is None + assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # The mapped protein comes off the protein-level member allele's hgvs_p, not the measured coding allele. + assert record_out.protein_level_hgvs == HgvsField( + hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr" + ) + + +@pytest.mark.integration +def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db_with_score_set): + """Protein assay: the mapped assay-level string is a p. expression, parsed into a protein block; the + digest comes off the authoritative protein allele.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele( + session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" + ) + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level_digest == "prot-digest" + assert record_out.clingen_allele_id == "PA9" + block = HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + # For a protein assay the measured allele *is* the protein one, so the two mapped fields coincide. + assert record_out.assay_level_hgvs == block + assert record_out.protein_level_hgvs == block + + +@pytest.mark.integration +def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_score_set): + """The canonical string is always present; a splice/multivariant expression simply carries no block.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="NM_000546.6:c.[197A>G;472T>C]", hgvs_splice="c.122-6T>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="c.76_78del") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + # Multivariant, intronic, and indel expressions are all string-only (no position/ref/alt block). + assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") + assert record_out.hgvs_nt.position is None + assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") + assert record_out.assay_level_hgvs == HgvsField(hgvs="c.76_78del") + + +@pytest.mark.integration +def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_score_set): + """A variant with no live mapping record keeps its submitted HGVS + score but null mapped fields.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.5, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.variant_urn == variant.urn + assert record_out.score == 0.5 + assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert record_out.consequence is None + assert record_out.clingen_allele_id is None + assert record_out.assay_level_digest is None + assert record_out.assay_level_hgvs is None + assert record_out.protein_level_hgvs is None + + +@pytest.mark.integration +def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): + """A mapped variant with no live VEP consequence has consequence None but keeps its digest + mapped HGVS.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.consequence is None + assert record_out.assay_level_digest == "cdna-digest" + assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + + +@pytest.mark.integration +def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): + """Records come back ordered by the integer after '#', not insertion or lexical order.""" + score_set = setup_lib_db_with_score_set + for suffix in (10, 2, 1): + _variant(session, score_set, suffix, score=float(suffix)) + + records = get_lean_score_set_variants(session, score_set) + + assert [r.variant_urn for r in records] == [f"{score_set.urn}#{n}" for n in (1, 2, 10)] + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_db_with_score_set): + """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at + a past instant, while the immutable submitted HGVS and score are unaffected.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + + old_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", valid_from=T0) + old_allele = _allele(session, "old-digest", level="cdna") + _link(session, old_record, old_allele, is_authoritative=True, valid_from=T0) + old_record.retire(session, at=T1) # cascades to the link via __retire_cascade__ + + new_record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.500A>T", valid_from=T1) + new_allele = _allele(session, "new-digest", level="cdna") + _link(session, new_record, new_allele, is_authoritative=True, valid_from=T1) + session.commit() + + [current] = get_lean_score_set_variants(session, score_set) + assert current.assay_level_digest == "new-digest" + assert current.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + + [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) + assert historical.assay_level_digest == "old-digest" + assert historical.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + # Submitted HGVS and score are immutable — unaffected by as_of. + assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") + assert historical.score == -2.3 + + # Before anything was mapped, the variant still appears with null mapped fields. + [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) + assert pre_mapping.assay_level_digest is None + assert pre_mapping.assay_level_hgvs is None diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 4a896c3cb..055d6bbb2 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -24,9 +24,14 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.allele import Allele from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate from tests.helpers.constants import ( @@ -4559,3 +4564,164 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( f"No gnomad variants matching the provided filters associated with score set URN {score_set['urn']} were found" in response_data["detail"] ) + + +def _seed_lean_mapping(session, variant_urn): + """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose + authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped={"type": "Allele"}, clingen_allele_id="CA123") + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped={"type": "Allele"}, hgvs_p="NP_000537.3:p.Ala406Thr" + ) + session.add_all([measured, protein]) + session.commit() + + session.add_all( + [ + MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_lean_mapping(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + records = response.json() + # All three variants are returned, ordered by variant number. + assert [r["variantUrn"] for r in records] == [f"{score_set['urn']}#{n}" for n in (1, 2, 3)] + for record in records: + LeanVariant.model_validate_json(json.dumps(record)) + + mapped = records[0] + assert mapped["score"] == 0.3 + assert mapped["consequence"] == "missense_variant" + assert mapped["clingenAlleleId"] == "CA123" + assert mapped["assayLevelDigest"] == "cdna-digest" + # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. + assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} + assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} + # Mapped assay-level HGVS (reference frame) and the mapped protein representation. + assert mapped["assayLevelHgvs"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["proteinLevelHgvs"] == { + "hgvs": "NP_000537.3:p.Ala406Thr", + "position": 406, + "ref": "Ala", + "alt": "Thr", + } + + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none). + unmapped = records[1] + assert unmapped["score"] == 1.0 + assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevelHgvs"): + assert omitted not in unmapped + + +def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variants") + assert response.status_code == 404 + + +def test_get_lean_variants_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Default is current — the resolved content-time is echoed for the client. + current = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + + # An explicit as_of is accepted and echoed back; before mapping exists everything is unmapped. + historical = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants", params={"as_of": "2020-01-01T00:00:00Z"}) + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert all("assayLevelDigest" not in record for record in historical.json()) + + +def test_get_lean_variants_other_user_cannot_read_private(client, session, data_provider, data_files, setup_router_db): + """The endpoint gates on READ: a non-owner cannot read another user's private (unpublished) set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + assert f"score set with URN '{score_set['urn']}' not found" in response.json()["detail"] + + +def test_get_lean_variants_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 404 + + +def test_get_lean_variants_anonymous_can_read_published( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """A published score set is world-readable, so an anonymous caller gets the full lean set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + published = publish_score_set(client, score_set["urn"]) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 + + +def test_get_lean_variants_admin_can_read_private( + client, session, data_provider, data_files, setup_router_db, admin_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(admin_app_overrides): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variants") + + assert response.status_code == 200 + assert len(response.json()) == 3 From f563760c0330d97d747227af596cc572eeedb7c3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 11:07:47 -0700 Subject: [PATCH 54/93] refactor(valid-time): add ValidTime.live_at for uniform temporal filtering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a live_at(as_of) hybrid_method to the ValidTime mixin — the one-call form of `as_of(ts) if ts is not None else current`. It is alias-aware (like as_of/current, its column refs adapt to an aliased() class), so a per-layer subquery can filter an aliased ValidTime table directly rather than hand-rolling a valid_to predicate. Migrate the repeated ternary in alleles.py to it. Covered by TestLiveAt: instance both branches, expression agrees with current/as_of, and alias-awareness. --- src/mavedb/db/mixins.py | 23 +++++++++++++++++++-- src/mavedb/lib/alleles.py | 17 ++++++++-------- tests/db/test_mixins.py | 43 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/mavedb/db/mixins.py b/src/mavedb/db/mixins.py index 4bc5c1ca9..c9b17f4b3 100644 --- a/src/mavedb/db/mixins.py +++ b/src/mavedb/db/mixins.py @@ -3,8 +3,9 @@ from datetime import datetime from typing import ClassVar, Optional, Sequence, TypeVar -from sqlalchemy import ColumnElement, DateTime, Update, func, inspect as sa_inspect, select, update -from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import ColumnElement, DateTime, Update, func, select, update +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property from sqlalchemy.orm import Mapped, Session, mapped_column T = TypeVar("T", bound="ValidTime") @@ -97,6 +98,24 @@ def as_of(cls, ts: datetime) -> ColumnElement[bool]: """Filter clause selecting the rows live at ``ts`` (half-open ``[valid_from, valid_to)``).""" return (cls.valid_from <= ts) & (cls.valid_to.is_(None) | (cls.valid_to > ts)) + @hybrid_method + def live_at(self, as_of: Optional[datetime]) -> bool: + """Live at ``as_of``, or currently live when ``as_of`` is ``None``. + + The one-call form of the ``as_of(ts) if ts is not None else current`` branch every temporal + read otherwise repeats. Same result as :attr:`current` / :meth:`as_of`, but usable uniformly on + the class, an instance, and an ``aliased()`` class (SQLAlchemy adapts the column refs to it).""" + if as_of is None: + return self.valid_to is None + return self.valid_from <= as_of and (self.valid_to is None or self.valid_to > as_of) + + @live_at.expression + @classmethod + def _live_at_expression(cls, as_of: Optional[datetime]) -> ColumnElement[bool]: + if as_of is None: + return cls.valid_to.is_(None) + return (cls.valid_from <= as_of) & (cls.valid_to.is_(None) | (cls.valid_to > as_of)) + def retire(self, session: Optional[Session] = None, at: Optional[datetime] = None) -> None: """Close this row's validity window, cascading to the live child links named in ``__retire_cascade__``. Idempotent — an already-closed row keeps its original valid_to. diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index 7a7213fd1..bf6a340d4 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -35,10 +35,9 @@ def get_live_record_allele_links( applies the same half-open predicate to both the record and the links, so the set is evaluated at one instant. Returns ``[]`` when the variant has no live record. """ - record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - - record_id = db.scalar(select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(record_live)) + record_id = db.scalar( + select(MappingRecord.id).where(MappingRecord.variant_id == variant_id).where(MappingRecord.live_at(as_of)) + ) if record_id is None: return [] @@ -46,7 +45,7 @@ def get_live_record_allele_links( db.scalars( select(MappingRecordAllele) .where(MappingRecordAllele.mapping_record_id == record_id) - .where(link_live) + .where(MappingRecordAllele.live_at(as_of)) .options(joinedload(MappingRecordAllele.allele)) ).all() ) @@ -69,10 +68,10 @@ def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[date instant. The retire-cascade invariant (a live link implies a live record) holds under ``as_of`` too, so filtering the links alone is sufficient. """ - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - record_ids = db.scalars( - select(MappingRecordAllele.mapping_record_id).where(MappingRecordAllele.allele_id == allele_id).where(link_live) + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id == allele_id) + .where(MappingRecordAllele.live_at(as_of)) ).all() if not record_ids: return [] @@ -82,7 +81,7 @@ def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[date select(Allele) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) - .where(link_live) + .where(MappingRecordAllele.live_at(as_of)) .distinct() ).all() ) diff --git a/tests/db/test_mixins.py b/tests/db/test_mixins.py index ad0333a5d..938d2062e 100644 --- a/tests/db/test_mixins.py +++ b/tests/db/test_mixins.py @@ -15,7 +15,7 @@ import pytest from sqlalchemy import Column, ForeignKey, Index, Integer, select, text from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import declarative_base, relationship +from sqlalchemy.orm import aliased, declarative_base, relationship from mavedb.db.mixins import ValidTime @@ -120,6 +120,47 @@ def live_ids(ts): assert live_ids(T2 + timedelta(days=1)) == [] # after the window closes +class TestLiveAt: + """``live_at(as_of)`` is the one-call ``as_of(ts) if ts else current`` — same result, but it also + works on an ``aliased()`` entity, which the raw ternary at call sites does too, but only clumsily.""" + + def test_live_at_none_is_current_on_instance(self, session): + live = _parent(session, 1) + retired = _parent(session, 2) + retired.retire(session, at=T1) + assert live.live_at(None) is True + assert retired.live_at(None) is False + + def test_live_at_ts_matches_the_window_on_instance(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + assert p.live_at(T0 - timedelta(days=1)) is False # before the window opens + assert p.live_at(T0) is True # at valid_from (inclusive) + assert p.live_at(T1) is True # inside the window + assert p.live_at(T2) is False # at valid_to (exclusive) + + def test_live_at_expression_agrees_with_current_and_as_of(self, session): + p = _parent(session, 1, valid_from=T0) + p.retire(session, at=T2) + session.commit() + + def ids(pred): + return [r.id for r in session.scalars(select(_VtParent).where(pred)).all()] + + assert ids(_VtParent.live_at(None)) == ids(_VtParent.current) == [] # retired -> not current + assert ids(_VtParent.live_at(T1)) == ids(_VtParent.as_of(T1)) == [p.id] + + def test_live_at_is_alias_aware(self, session): + # The point of live_at: it scopes to the alias, not the base table (a plain classmethod on an + # aliased entity is the trap this avoids). Both the compiled SQL and a query through the alias. + p = _parent(session, 1, valid_from=T0) + session.commit() + alias = aliased(_VtParent, name="pa") + assert "pa." in str(alias.live_at(T1)) + rows = session.scalars(select(alias).where(alias.live_at(T1))).all() + assert [r.id for r in rows] == [p.id] + + class TestRetire: def test_retire_closes_valid_to(self, session): c = _child(session, _parent(session, 1), 1) From a5f3da700def00da47586ab1bb0df6159d19090b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 1 Jul 2026 11:09:09 -0700 Subject: [PATCH 55/93] refactor(lean_variants): use live_at helper and join-based protein hgvs Rework get_lean_score_set_variants to simplify ValidTime handling and the post-mapped protein HGVS projection. - Replace the inline record/link/vep liveness predicates with a shared live_at(as_of) helper on each ValidTime model, removing the ad-hoc and_/or_ valid_from/valid_to comparisons (and the now-unused or_ import) - Swap the correlated scalar subquery for protein_hgvs for a single DISTINCT ON subquery scoped to the score set and joined once, so its cost scales with the response rather than per-row correlation - Expand comments to document why live_at rides in each ON clause (a WHERE would collapse the LEFT joins to inner joins for unmapped variants) --- src/mavedb/lib/lean_variants.py | 77 ++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py index f0949e614..4ac9df53f 100644 --- a/src/mavedb/lib/lean_variants.py +++ b/src/mavedb/lib/lean_variants.py @@ -22,7 +22,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy import Integer, and_, cast, func, select from sqlalchemy.orm import Session, aliased from mavedb.lib.hgvs import parse_simple_substitution @@ -92,32 +92,34 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # Content valid-time: each ValidTime layer is evaluated at the same instant, or at the current tail. - record_live = MappingRecord.as_of(as_of) if as_of is not None else MappingRecord.current - link_live = MappingRecordAllele.as_of(as_of) if as_of is not None else MappingRecordAllele.current - vep_live = VepAlleleConsequence.as_of(as_of) if as_of is not None else VepAlleleConsequence.current - - # The post-mapped protein HGVS is the forward-translated protein consequence, stored in the hgvs_p - # column of the record's protein-level allele. Pull it as a correlated scalar subquery so the main - # query stays one row per variant. It is null when no forward-translated protein is available. For - # a protein assay this is the measured allele, so it coincides with assay_level_hgvs. + # Per record, the forward-translated protein HGVS (hgvs_p) of its protein-level allele. prot_link = aliased(MappingRecordAllele) prot_allele = aliased(Allele) - prot_link_live = ( - and_(prot_link.valid_from <= as_of, or_(prot_link.valid_to.is_(None), prot_link.valid_to > as_of)) - if as_of is not None - else prot_link.valid_to.is_(None) - ) - protein_hgvs_subquery = ( - select(prot_allele.hgvs_p) - .join(prot_link, prot_link.allele_id == prot_allele.id) - .where(prot_link.mapping_record_id == MappingRecord.id) - .where(prot_link_live) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_allele = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the subquery (next). + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the subquery's cost scales with the response. + .where(prot_variant.score_set_id == score_set.id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. .where(prot_allele.level == AnnotationLayer.protein.value) - .order_by(prot_allele.id) - .limit(1) - .correlate(MappingRecord) - .scalar_subquery() + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + .subquery() ) statement = ( @@ -128,26 +130,39 @@ def get_lean_score_set_variants( Variant.hgvs_pro, Variant.hgvs_splice, MappingRecord.hgvs_assay_level, - protein_hgvs_subquery.label("protein_hgvs"), + protein_allele.c.protein_hgvs.label("protein_hgvs"), Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, ) - .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, record_live)) - # Only the authoritative (measured) allele is needed, so the link join stays 1:1 with the - # variant — one authoritative link per live record (the invariant the mapping job upholds). + # Variant -> its live mapping record. LEFT join so unmapped variants stay (with null mapped fields) + # for the table + score histogram. live_at rides in the ON clause, never a WHERE: a WHERE would + # reject the null-record row an outer join makes for an unmapped variant, silently collapsing the + # LEFT join to an inner one. Same pattern on every join below. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + # The one authoritative (measured) allele. is_authoritative keeps this 1:1 with the variant + # (one authoritative link per live record — the invariant the mapping job upholds). .outerjoin( MappingRecordAllele, and_( MappingRecordAllele.mapping_record_id == MappingRecord.id, MappingRecordAllele.is_authoritative.is_(True), - link_live, + MappingRecordAllele.live_at(as_of), ), ) + # The authoritative allele row itself (digest, ClinGen id). No live_at: alleles are immutable and + # deduplicated, not ValidTime — the live link already establishes what applies. .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) - .outerjoin(VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, vep_live)) + # Its VEP consequence. + .outerjoin( + VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) + ) + # The protein subquery, matched back by record id (a globally-unique PK -> the right protein row, + # no cross-set risk). LEFT: some rows have no protein projection (UTR/intronic) -> null. + .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) + # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) - # Variant number (the integer after '#') is the natural table order; id breaks ties stably. + # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) From fc6c8ad70413e0454606a94b4b5584630b01e9df Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 6 Jul 2026 08:21:36 -0700 Subject: [PATCH 56/93] perf(lean-variants): materialize protein projection to avoid O(N^2) plan The protein-level HGVS projection was a plain subquery. Postgres badly under-counted its DISTINCT ON cardinality (est. ~5 vs ~2.8k actual) and buried it on the inner side of a nested loop, re-running the whole score-set-scoped projection once per variant. That is O(N^2) in the score set size and made the whole-set variants query hang on large sets. Rendering it as a MATERIALIZED CTE forces single evaluation, collapsing the query back to O(N). Results are unchanged; only the plan differs. --- src/mavedb/lib/lean_variants.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/lean_variants.py index 4ac9df53f..134a4eae2 100644 --- a/src/mavedb/lib/lean_variants.py +++ b/src/mavedb/lib/lean_variants.py @@ -119,7 +119,12 @@ def get_lean_score_set_variants( # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives # per record — inert today (only one protein allele exists). .order_by(prot_link.mapping_record_id, prot_allele.id) - .subquery() + # MATERIALIZED forces this to run exactly once. The planner badly under-counted the DISTINCT ON + # cardinality (est. ~5 vs ~2.8k actual) and, left as a plain subquery, buries it on the inner + # side of a nested loop, re-running the whole projection once per variant. That is O(N^2) in the + # score set size. Materializing collapses it back to O(N). + .cte("protein_allele") + .prefix_with("MATERIALIZED") ) statement = ( @@ -157,8 +162,9 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) - # The protein subquery, matched back by record id (a globally-unique PK -> the right protein row, - # no cross-set risk). LEFT: some rows have no protein projection (UTR/intronic) -> null. + # The protein projection (materialized above), matched back by record id (a globally-unique PK -> + # the right protein row, no cross-set risk). LEFT: some rows have no protein projection + # (UTR/intronic) -> null. .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) From b6ea2dcefa940741fe5d76e6384bfcc88bdb9798 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:44:38 -0700 Subject: [PATCH 57/93] refactor(score-sets): rename lean_variants module to score_set_variants The module serves the whole-set variant view for a score set, not a generic 'lean variant' concept; the new name pairs it with lib/score_sets.py and lib/alleles.py. Pure rename plus reference/docstring updates. --- src/mavedb/lib/{lean_variants.py => score_set_variants.py} | 0 src/mavedb/routers/score_sets.py | 2 +- src/mavedb/view_models/lean_variant.py | 2 +- .../lib/{test_lean_variants.py => test_score_set_variants.py} | 4 ++-- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/mavedb/lib/{lean_variants.py => score_set_variants.py} (100%) rename tests/lib/{test_lean_variants.py => test_score_set_variants.py} (99%) diff --git a/src/mavedb/lib/lean_variants.py b/src/mavedb/lib/score_set_variants.py similarity index 100% rename from src/mavedb/lib/lean_variants.py rename to src/mavedb/lib/score_set_variants.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 300191285..34ef50df2 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -41,7 +41,6 @@ find_or_create_doi_identifier, find_or_create_publication_identifier, ) -from mavedb.lib.lean_variants import get_lean_score_set_variants from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import ( correlation_id_for_context, @@ -50,6 +49,7 @@ ) from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.score_set_variants import get_lean_score_set_variants from mavedb.lib.score_sets import ( CLINVAR_NS_PATTERN, csv_data_to_df, diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 1820cfc71..029a217a6 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -1,6 +1,6 @@ """Response view models for the lean whole-set view (``GET /score-sets/{urn}/variants``). -The pydantic serialization boundary over the ``lib.lean_variants`` transit dataclasses. ``from_attributes`` +The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` coerce them (field names line up); aliases camelize per the shared base config, so clients see ``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot diff --git a/tests/lib/test_lean_variants.py b/tests/lib/test_score_set_variants.py similarity index 99% rename from tests/lib/test_lean_variants.py rename to tests/lib/test_score_set_variants.py index 79d630815..22f4f43d0 100644 --- a/tests/lib/test_lean_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -1,5 +1,5 @@ # ruff: noqa: E402 -"""Integration tests for the lean whole-set view assembly (``lib/lean_variants.py``). +"""Integration tests for the lean whole-set view assembly (``lib/score_set_variants.py``). These pin the one-row-per-variant assembly: the submitted HGVS come off the variant, the mapped assay-level HGVS off the live mapping record, the digest/ClinGen id/VEP consequence off the @@ -15,7 +15,7 @@ pytest.importorskip("psycopg2") -from mavedb.lib.lean_variants import HgvsField, get_lean_score_set_variants +from mavedb.lib.score_set_variants import HgvsField, get_lean_score_set_variants from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele From 6a6112af4ac636614eb32aebf1d961e38561d307 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:46:37 -0700 Subject: [PATCH 58/93] feat(variants): variant detail envelope for GET /variants/{urn} Replace the flat measurement response with a two-tier envelope: flat assay-level fields (assay level, target/reference HGVS, digest, ClinGen id), per-calibration classifications, the spec-pure Cat-VRS molecular representation, and a digest-keyed annotation map (VEP/gnomAD/ClinVar). Adds ?as_of= reconstruction of the molecular layer (scores/classifications carry no ValidTime, so they are as-of-invariant) and folds in supersession self-describe (is_current/superseded_by) on this single-resource route. New lib/allele_annotations.py (digest-keyed annotation assembler, reused by the allele page) and lib/variant_detail.py (envelope composition); view models split into allele_annotation + variant_detail. Model columns gain Mapped[...] typing to back the annotation reads. Deletes the dead VariantEffectMeasurementWithScoreSet. --- src/mavedb/lib/allele_annotations.py | 151 ++++++++++++ src/mavedb/lib/variant_detail.py | 180 ++++++++++++++ src/mavedb/models/allele.py | 2 +- src/mavedb/models/clinical_control.py | 16 +- src/mavedb/models/gnomad_variant.py | 18 +- src/mavedb/models/vep_allele_consequence.py | 4 +- src/mavedb/routers/variants.py | 62 ++++- src/mavedb/view_models/allele_annotation.py | 60 +++++ src/mavedb/view_models/variant.py | 22 +- src/mavedb/view_models/variant_detail.py | 63 +++++ tests/lib/test_allele_annotations.py | 162 +++++++++++++ tests/lib/test_variant_detail.py | 248 ++++++++++++++++++++ tests/routers/test_variant.py | 172 ++++++++++++++ 13 files changed, 1116 insertions(+), 44 deletions(-) create mode 100644 src/mavedb/lib/allele_annotations.py create mode 100644 src/mavedb/lib/variant_detail.py create mode 100644 src/mavedb/view_models/allele_annotation.py create mode 100644 src/mavedb/view_models/variant_detail.py create mode 100644 tests/lib/test_allele_annotations.py create mode 100644 tests/lib/test_variant_detail.py create mode 100644 tests/routers/test_variant.py diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py new file mode 100644 index 000000000..1b6cf112b --- /dev/null +++ b/src/mavedb/lib/allele_annotations.py @@ -0,0 +1,151 @@ +"""Digest-keyed annotation assembly over the deduplicated allele model. + +Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar +assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride +annotations *alongside* the spec-pure Cat-VRS members (design §8): Cat-VRS itself carries no +annotation slot, so annotations travel beside it in a flat digest-keyed map. + +All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on +``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past +instant via ``live_at``, defaulting to the currently-live rows. Absence is normal and encoded as +``None``/empty — annotations are sparse and computed at all levels with no fixed level→source +mapping (D44/D44b): a genomic allele *may* carry VEP, a coding allele *may* match gnomAD. + +VEP and gnomAD are one-live-per-allele, so each is at most one value; ClinVar is multi-live (one +live link per release), so it is a list. + +This is the shared assembler for both ``GET /variants/{urn}`` (all of a variant's alleles) and +``GET /alleles/{digest}`` (a single allele) — pass whichever allele set the caller has in hand. +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, Sequence + +from sqlalchemy import Result, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass(frozen=True) +class VepAnnotation: + """A VEP most-severe functional consequence and the Ensembl release it was resolved under.""" + + consequence: Optional[str] + source_version: str + + +@dataclass(frozen=True) +class GnomadAnnotation: + """gnomAD population frequency for an allele, from its live gnomAD link.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] + db_version: str + db_identifier: str + + +@dataclass(frozen=True) +class ClinvarAnnotation: + """One ClinVar assertion for an allele (one per live release — an allele may carry several).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] + clinvar_allele_id: str + db_version: str + + +@dataclass +class AlleleAnnotations: + """The external annotations for a single allele. Every field is sparse — a source with no data + for the allele is ``None`` (vep/gnomad) or an empty list (clinvar).""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = field(default_factory=list) + + +def get_allele_annotations( + db: Session, alleles: Sequence[Allele], *, as_of: Optional[datetime] = None +) -> dict[str, AlleleAnnotations]: + """Assemble the digest-keyed annotation map for ``alleles``. + + Returns one :class:`AlleleAnnotations` per distinct ``vrs_digest`` present in ``alleles``. An + allele with no annotations still gets an (empty) entry, so the map's keys mirror the alleles + handed in — the envelope can join every Cat-VRS member to a (possibly empty) annotation block. + ``as_of`` reconstructs each source at a past instant; it defaults to the currently-live rows. + """ + # id -> digest for the alleles we were handed; digest is the map key the envelope joins on. + digest_by_id = {allele.id: allele.vrs_digest for allele in alleles} + annotations: dict[str, AlleleAnnotations] = { + digest: AlleleAnnotations() for digest in digest_by_id.values() if digest is not None + } + if not digest_by_id: + return annotations + + allele_ids = list(digest_by_id.keys()) + + # VEP — at most one live consequence per allele. + vep_rows = db.execute( + select( + VepAlleleConsequence.allele_id, + VepAlleleConsequence.functional_consequence, + VepAlleleConsequence.source_version, + ) + .where(VepAlleleConsequence.allele_id.in_(allele_ids)) + .where(VepAlleleConsequence.live_at(as_of)) + ).tuples() + for allele_id, consequence, source_version in vep_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].vep = VepAnnotation(consequence=consequence, source_version=source_version) + + # gnomAD — at most one live link per allele, resolving to a frequency record. + gnomad_rows: Result[tuple[int, GnomADVariant]] = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + ) + for allele_id, gnomad_variant in gnomad_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].gnomad = GnomadAnnotation( + allele_frequency=gnomad_variant.allele_frequency, + allele_count=gnomad_variant.allele_count, + allele_number=gnomad_variant.allele_number, + faf95_max=gnomad_variant.faf95_max, + db_version=gnomad_variant.db_version, + db_identifier=gnomad_variant.db_identifier, + ) + + # ClinVar — multi-live (one live link per release), so each allele accumulates a list. + clinvar_rows: Result[tuple[int, ClinvarControl]] = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + ) + for allele_id, clinvar_control in clinvar_rows: + digest = digest_by_id[allele_id] + if digest is not None: + annotations[digest].clinvar.append( + ClinvarAnnotation( + clinical_significance=clinvar_control.clinical_significance, + clinical_review_status=clinvar_control.clinical_review_status, + clinvar_variation_id=clinvar_control.clinvar_variation_id, + clinvar_allele_id=clinvar_control.db_identifier, + db_version=clinvar_control.db_version, + ) + ) + + return annotations diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py new file mode 100644 index 000000000..09e48a072 --- /dev/null +++ b/src/mavedb/lib/variant_detail.py @@ -0,0 +1,180 @@ +"""The assayed variant-detail view backing ``GET /variants/{urn}``. + +Assembles the two-tier envelope for a single variant: flat, UI-ergonomic assay fields (the +assay-level HGVS pair, digest, ClinGen id) plus the spec-pure GA4GH ``CategoricalVariant`` (built +on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — +per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also +carries the per-calibration functional ``classifications`` the variant falls into, and its version +standing (``is_current`` / ``superseded_by``) so a superseded variant self-describes rather than +reading as current (design §9.2). + +Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the +VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is +not ``ValidTime``) and calibrations carry no ``ValidTime`` at all (they are frozen/append-only), so +scores and classifications are as-of-invariant and returned as they stand now. ``as_of`` defaults to +the currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class VariantClassificationRecord: + """One functional classification the variant falls into, tagged with its calibration context. + + ``calibration_id`` / ``primary`` locate the classification among the score set's (possibly + several) calibrations — ``primary`` is the UI default. ``classification`` is the ORM + :class:`ScoreCalibrationFunctionalClassification` (functional call, ACMG, oddspath), serialized + at the view-model boundary. + """ + + calibration_id: int + primary: bool + classification: ScoreCalibrationFunctionalClassification + + +@dataclass(frozen=True) +class VariantDetail: + """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" + + urn: str + scores: Optional[dict[str, Any]] + counts: Optional[dict[str, Any]] + classifications: list[VariantClassificationRecord] + + # Flat, UI-ergonomic assay-level fields. + assay_level: Optional[str] + target_hgvs: Optional[str] # submitted, target/assay coordinates + reference_hgvs: Optional[str] # mapped, reference coordinates, assay level + assay_level_digest: Optional[str] + clingen_allele_id: Optional[str] + + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside. + molecular_representation: Optional[dict[str, Any]] + mode: Optional[str] # CatVrsMode value — projection | reverse_translation + member_relations: dict[str, str] # member vrs_digest -> relation (member -> defining) + + # External annotations, keyed by VRS digest, joined to the Cat-VRS members. + annotations: dict[str, AlleleAnnotations] + + # Version standing — self-descriptive for a superseded variant. + is_current: bool + superseded_by: Optional[str] # URN of the superseding variant's score set, if any (and readable) + + +def _classifications_for_variant( + db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] +) -> list[VariantClassificationRecord]: + """The functional classifications the variant belongs to (membership is materialized on the + calibration→classification→variant m2m). One row per calibration that classifies the variant; + the primary calibration sorts first. ``visible_calibration_ids`` restricts to calibrations the + caller resolved as readable (``None`` = no restriction, for lib-level use).""" + statement = ( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration.primary, ScoreCalibration.id) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + .order_by(ScoreCalibration.primary.desc(), ScoreCalibration.id) + ) + if visible_calibration_ids is not None: + statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) + + return [ + VariantClassificationRecord(calibration_id=calibration_id, primary=primary, classification=classification) + for classification, primary, calibration_id in db.execute(statement) + ] + + +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: + """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, + otherwise the nucleotide expression (genomic or coding share ``hgvs_nt``).""" + if assay_level == AnnotationLayer.protein.value: + return variant.hgvs_pro + return variant.hgvs_nt + + +def get_variant_detail( + db: Session, + variant: Variant, + *, + superseding_score_set: Optional[ScoreSet] = None, + visible_calibration_ids: Optional[set[int]] = None, + as_of: Optional[datetime] = None, +) -> VariantDetail: + """Assemble the variant-detail envelope for ``variant``. + + ``superseding_score_set`` is the newer version the caller has already resolved for visibility + (blanked when the user cannot read it, mirroring ``fetch_score_set_by_urn``); its presence drives + ``is_current``/``superseded_by``. ``visible_calibration_ids`` restricts classifications to + readable calibrations. ``as_of`` reconstructs the molecular layer (Cat-VRS membership + + annotations); scores and classifications are as-of-invariant. See the module docstring. + """ + data = variant.data if isinstance(variant.data, dict) else {} + scores = data.get("score_data") + counts = data.get("count_data") + + # The live mapping record supplies the assay level and the mapped (reference-frame) assay HGVS. + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + assay_level = record.assay_level if record is not None else None + reference_hgvs = record.hgvs_assay_level if record is not None else None + + # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all + # linked alleles seed the digest-keyed annotation map. + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + authoritative = next((link for link in links if link.is_authoritative), None) + assay_level_digest = authoritative.allele.vrs_digest if authoritative is not None else None + clingen_allele_id = authoritative.allele.clingen_allele_id if authoritative is not None else None + + annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) + + # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). + transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) + if transit is not None: + molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) + mode: Optional[str] = transit.mode.value + member_relations = {digest: relation.value for digest, relation in transit.member_relations.items()} + else: + molecular_representation = None + mode = None + member_relations = {} + + return VariantDetail( + urn=variant.urn or "", + scores=scores, + counts=counts, + classifications=_classifications_for_variant(db, variant, visible_calibration_ids=visible_calibration_ids), + assay_level=assay_level, + target_hgvs=_submitted_assay_level_hgvs(variant, assay_level), + reference_hgvs=reference_hgvs, + assay_level_digest=assay_level_digest, + clingen_allele_id=clingen_allele_id, + molecular_representation=molecular_representation, + mode=mode, + member_relations=member_relations, + annotations=annotations, + is_current=superseding_score_set is None, + superseded_by=superseding_score_set.urn if superseding_score_set is not None else None, + ) diff --git a/src/mavedb/models/allele.py b/src/mavedb/models/allele.py index a57c4c802..67eadd519 100644 --- a/src/mavedb/models/allele.py +++ b/src/mavedb/models/allele.py @@ -18,7 +18,7 @@ class Allele(Base): id: Mapped[int] = Column(Integer, primary_key=True) - vrs_digest = Column(String, nullable=False) + vrs_digest: Mapped[str] = Column(String, nullable=False) level = Column(String(length=16), nullable=False) hgvs_g = Column(String, nullable=True) diff --git a/src/mavedb/models/clinical_control.py b/src/mavedb/models/clinical_control.py index 694aad58e..388444a09 100644 --- a/src/mavedb/models/clinical_control.py +++ b/src/mavedb/models/clinical_control.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, relationship @@ -22,18 +22,18 @@ class ClinvarControl(Base): id: Mapped[int] = Column(Integer, primary_key=True) - gene_symbol = Column(String, nullable=False, index=True) + gene_symbol: Mapped[str] = Column(String, nullable=False, index=True) - clinical_significance = Column(String, nullable=False) - clinical_review_status = Column(String, nullable=False) + clinical_significance: Mapped[str] = Column(String, nullable=False) + clinical_review_status: Mapped[str] = Column(String, nullable=False) - db_name = Column(String, nullable=False, index=True) + db_name: Mapped[str] = Column(String, nullable=False, index=True) # ClinVar Allele ID (row level link). - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False, index=True) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False, index=True) # ClinVar Variation ID (variation level link). - clinvar_variation_id = Column(String, nullable=True) + clinvar_variation_id: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/models/gnomad_variant.py b/src/mavedb/models/gnomad_variant.py index bfc7c8026..bb9a1faf6 100644 --- a/src/mavedb/models/gnomad_variant.py +++ b/src/mavedb/models/gnomad_variant.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, Float, Integer, String from sqlalchemy.orm import Mapped, relationship @@ -17,16 +17,16 @@ class GnomADVariant(Base): id: Mapped[int] = Column(Integer, primary_key=True) - db_name = Column(String, nullable=False) - db_identifier = Column(String, nullable=False, index=True) - db_version = Column(String, nullable=False) + db_name: Mapped[str] = Column(String, nullable=False) + db_identifier: Mapped[str] = Column(String, nullable=False, index=True) + db_version: Mapped[str] = Column(String, nullable=False) - allele_count = Column(Integer, nullable=False) - allele_number = Column(Integer, nullable=False) - allele_frequency = Column(Float, nullable=False) + allele_count: Mapped[int] = Column(Integer, nullable=False) + allele_number: Mapped[int] = Column(Integer, nullable=False) + allele_frequency: Mapped[float] = Column(Float, nullable=False) - faf95_max = Column(Float, nullable=True) - faf95_max_ancestry = Column(String, nullable=True) + faf95_max: Mapped[Optional[float]] = Column(Float, nullable=True) + faf95_max_ancestry: Mapped[Optional[str]] = Column(String, nullable=True) creation_date = Column(Date, nullable=False, default=date.today) modification_date = Column(Date, nullable=False, default=date.today, onupdate=date.today) diff --git a/src/mavedb/models/vep_allele_consequence.py b/src/mavedb/models/vep_allele_consequence.py index de41230a5..4133a1231 100644 --- a/src/mavedb/models/vep_allele_consequence.py +++ b/src/mavedb/models/vep_allele_consequence.py @@ -1,5 +1,5 @@ from datetime import date -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from sqlalchemy import Column, Date, ForeignKey, Index, Integer, String, text from sqlalchemy.orm import Mapped, relationship @@ -52,7 +52,7 @@ class VepAlleleConsequence(ValidTime, Base): ForeignKey("alleles.id", ondelete="RESTRICT"), nullable=False, ) - functional_consequence = Column(String, nullable=True) + functional_consequence: Mapped[Optional[str]] = Column(String, nullable=True) source_version: Mapped[str] = Column(String, nullable=False) access_date: Mapped[date] = Column(Date, nullable=False) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..97352a8ec 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,8 +1,10 @@ import itertools import logging import re +from datetime import datetime +from typing import Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query, Response from fastapi.exceptions import HTTPException from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound @@ -15,6 +17,7 @@ from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.variant_detail import get_variant_detail from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -28,8 +31,8 @@ from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, - VariantEffectMeasurementWithScoreSet, ) +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -437,19 +440,37 @@ def lookup_variants( @router.get( "/variants/{urn}", status_code=200, - response_model=VariantEffectMeasurementWithScoreSet, + response_model=VariantDetail, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, response_model_exclude_none=True, - summary="Fetch variant by URN", + summary="Fetch assayed variant detail by URN", ) -def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user)): - """ - Fetch a single variant by URN. +def get_variant( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """Fetch the two-tier detail envelope for a single assayed variant by URN. + + Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a + digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is + the citable unit) but self-describes via isCurrent/supersededBy rather than reading as current. """ - save_to_logging_context({"requested_resource": urn}) + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" try: - query = db.query(Variant).filter(Variant.urn == urn) - variant = query.one_or_none() + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() except MultipleResultsFound: logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") @@ -459,4 +480,23 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") assert_permission(user_data, variant.score_set, Action.READ) - return variant + + # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn + # does — a newer version the user cannot read is not leaked (the variant then reads as current). + superseding = variant.score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + visible_calibration_ids = { + sc.id + for sc in variant.score_set.score_calibrations + if sc.id is not None and has_permission(user_data, sc, Action.READ).permitted + } + + return get_variant_detail( + db, + variant, + superseding_score_set=superseding, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, + ) diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py new file mode 100644 index 000000000..b538d4639 --- /dev/null +++ b/src/mavedb/view_models/allele_annotation.py @@ -0,0 +1,60 @@ +"""Response view models for an allele's external annotations (VEP / gnomAD / ClinVar). + +The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses +(``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and are the allele's own block +on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather than in either +one's file. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class VepAnnotation(BaseModel): + """VEP most-severe functional consequence and the Ensembl release it resolved under.""" + + consequence: Optional[str] = None + source_version: str + + class Config: + from_attributes = True + + +class GnomadAnnotation(BaseModel): + """gnomAD population frequency for an allele.""" + + allele_frequency: float + allele_count: int + allele_number: int + faf95_max: Optional[float] = None + db_version: str + db_identifier: str + + class Config: + from_attributes = True + + +class ClinvarAnnotation(BaseModel): + """One ClinVar assertion for an allele (an allele may carry one per release).""" + + clinical_significance: str + clinical_review_status: str + clinvar_variation_id: Optional[str] = None + clinvar_allele_id: str + db_version: str + + class Config: + from_attributes = True + + +class AlleleAnnotations(BaseModel): + """The external annotations for one allele, sparse — each source absent unless it has data.""" + + vep: Optional[VepAnnotation] = None + gnomad: Optional[GnomadAnnotation] = None + clinvar: list[ClinvarAnnotation] = [] + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index b01b01832..ad02023df 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -9,7 +9,7 @@ from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant if TYPE_CHECKING: - from mavedb.view_models.score_set import ScoreSet, ShortScoreSet + from mavedb.view_models.score_set import ShortScoreSet class VariantEffectMeasurementBase(BaseModel): @@ -76,13 +76,6 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): pass -class VariantEffectMeasurementWithScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and score set""" - - score_set: "ScoreSet" - mapped_variants: list[MappedVariant] - - class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): """Variant effect measurement view model with mapped variants and a limited set of score set details""" @@ -96,8 +89,11 @@ class ClingenAlleleIdVariantLookupsRequest(BaseModel): clingen_allele_ids: list[str] -class Variant(BaseModel): - """View model for a variant, defined by its ClinGen allele id, with associated variant effect measurements""" +class ClingenAlleleVariants(BaseModel): + """The assayed variants sharing one ClinGen allele id, with their variant effect measurements. + + An allele-side grouping (keyed by ClinGen allele id), it is the mapped-allele view of "which + measurements share this molecular identity," not the assayed variant itself.""" clingen_allele_id: str variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] @@ -107,6 +103,6 @@ class ClingenAlleleIdVariantLookupResponse(BaseModel): """Response model for a variant lookup by ClinGen allele ID""" clingen_allele_id: str - exact_match: Optional[Variant] = None - equivalent_nt: list[Variant] = [] - equivalent_aa: list[Variant] = [] + exact_match: Optional[ClingenAlleleVariants] = None + equivalent_nt: list[ClingenAlleleVariants] = [] + equivalent_aa: list[ClingenAlleleVariants] = [] diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py new file mode 100644 index 000000000..18a78f844 --- /dev/null +++ b/src/mavedb/view_models/variant_detail.py @@ -0,0 +1,63 @@ +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``, design §7.1). + +The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses +(``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its +own module — mirroring ``lib/variant_detail.py`` and the ``lean_variant`` precedent — so +``view_models/variant.py`` stays the core measurement + lookup file. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class VariantClassification(BaseModel): + """A functional classification the variant falls into, tagged with its calibration context. + + A score set may carry several calibrations, so a variant has one classification per calibration; + ``primary`` flags the UI default. The classifications are calibration-derived but as-of-invariant + (calibrations carry no valid-time), so they are always the current calibration state. + """ + + calibration_id: int + primary: bool + classification: SavedFunctionalClassification + + class Config: + from_attributes = True + + +class VariantDetail(BaseModel): + """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). + + Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair + is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` + (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS + digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` + /``supersededBy`` let a superseded variant self-describe. Absent fields are omitted. + """ + + urn: str + scores: Optional[dict[str, Any]] = None + counts: Optional[dict[str, Any]] = None + classifications: list[VariantClassification] = [] + + assay_level: Optional[str] = None + target_hgvs: Optional[str] = None + reference_hgvs: Optional[str] = None + assay_level_digest: Optional[str] = None + clingen_allele_id: Optional[str] = None + + molecular_representation: Optional[dict[str, Any]] = None + mode: Optional[str] = None + member_relations: dict[str, str] = {} + + annotations: dict[str, AlleleAnnotations] = {} + + is_current: bool + superseded_by: Optional[str] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_allele_annotations.py b/tests/lib/test_allele_annotations.py new file mode 100644 index 000000000..cedb4267c --- /dev/null +++ b/tests/lib/test_allele_annotations.py @@ -0,0 +1,162 @@ +# ruff: noqa: E402 +"""Integration tests for the digest-keyed annotation assembler (``lib/allele_annotations.py``). + +These pin: the map is keyed by ``vrs_digest`` and covers every allele handed in (an allele with no +annotations still gets an empty entry); VEP/gnomAD resolve to one value each while ClinVar is a list +(multi-live, one per release); absence is ``None``/empty; and ``as_of`` reconstructs each source at a +past instant over the immutable, content-addressed alleles. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_annotations import get_allele_annotations +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) + + +def _allele(session, digest, *, level="cdna"): + allele = Allele(vrs_digest=digest, level=level, post_mapped={"type": "Allele"}) + session.add(allele) + session.commit() + return allele + + +def _vep(session, allele, value, *, source_version="116", valid_from=None): + row = VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=value, + source_version=source_version, + access_date="2026-01-01", + valid_from=valid_from, + ) + session.add(row) + session.commit() + return row + + +def _gnomad(session, allele, *, allele_frequency=0.0123, db_version="4.1.0"): + gv = GnomADVariant( + db_name="gnomAD", + db_identifier=f"gnomad-{allele.vrs_digest}", + db_version=db_version, + allele_count=42, + allele_number=10000, + allele_frequency=allele_frequency, + faf95_max=0.05, + ) + session.add(gv) + session.commit() + link = GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gv.id) + session.add(link) + session.commit() + return gv + + +def _clinvar(session, allele, *, significance="Likely benign", db_version="11_2024", variation_id="12345"): + control = ClinvarControl( + db_identifier=f"cv-{db_version}-{allele.vrs_digest}", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status="criteria provided, multiple submitters, no conflicts", + db_name="ClinVar", + db_version=db_version, + clinvar_variation_id=variation_id, + ) + session.add(control) + session.commit() + link = ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=control.id) + session.add(link) + session.commit() + return control + + +@pytest.mark.integration +def test_empty_allele_list_returns_empty_map(session, setup_lib_db): + assert get_allele_annotations(session, []) == {} + + +@pytest.mark.integration +def test_allele_without_annotations_gets_an_empty_entry(session, setup_lib_db): + """The map's keys mirror the alleles handed in — a fully un-annotated allele still gets an + (empty) block, so the envelope can join every Cat-VRS member to something.""" + allele = _allele(session, "bare-digest") + + annotations = get_allele_annotations(session, [allele]) + + assert set(annotations) == {"bare-digest"} + block = annotations["bare-digest"] + assert block.vep is None + assert block.gnomad is None + assert block.clinvar == [] + + +@pytest.mark.integration +def test_all_three_sources_present(session, setup_lib_db): + allele = _allele(session, "rich-digest") + _vep(session, allele, "missense_variant") + _gnomad(session, allele, allele_frequency=0.0123) + _clinvar(session, allele, significance="Likely benign") + + block = get_allele_annotations(session, [allele])["rich-digest"] + + assert block.vep is not None and block.vep.consequence == "missense_variant" + assert block.vep.source_version == "116" + assert block.gnomad is not None and block.gnomad.allele_frequency == 0.0123 + assert block.gnomad.faf95_max == 0.05 + assert [c.clinical_significance for c in block.clinvar] == ["Likely benign"] + + +@pytest.mark.integration +def test_clinvar_is_multi_live_across_releases(session, setup_lib_db): + """ClinVar stacks one live link per release, so an allele can carry several assertions at once.""" + allele = _allele(session, "cv-digest") + _clinvar(session, allele, significance="Likely benign", db_version="11_2024") + _clinvar(session, allele, significance="Pathogenic", db_version="05_2025") + + block = get_allele_annotations(session, [allele])["cv-digest"] + + assert {(c.db_version, c.clinical_significance) for c in block.clinvar} == { + ("11_2024", "Likely benign"), + ("05_2025", "Pathogenic"), + } + + +@pytest.mark.integration +def test_map_is_keyed_by_digest_across_multiple_alleles(session, setup_lib_db): + a1 = _allele(session, "digest-1", level="cdna") + a2 = _allele(session, "digest-2", level="protein") + _vep(session, a1, "missense_variant") + _gnomad(session, a2) + + annotations = get_allele_annotations(session, [a1, a2]) + + assert set(annotations) == {"digest-1", "digest-2"} + assert annotations["digest-1"].vep is not None and annotations["digest-1"].gnomad is None + assert annotations["digest-2"].gnomad is not None and annotations["digest-2"].vep is None + + +@pytest.mark.integration +def test_as_of_reconstructs_the_historical_annotation(session, setup_lib_db): + """A changed VEP consequence supersedes the old row; as_of selects the value live at the instant.""" + allele = _allele(session, "vep-digest") + old = _vep(session, allele, "missense_variant", source_version="110", valid_from=T0) + old.retire(session, at=T1) + session.commit() + _vep(session, allele, "stop_gained", source_version="116", valid_from=T1) + + current = get_allele_annotations(session, [allele])["vep-digest"] + historical = get_allele_annotations(session, [allele], as_of=T1 - timedelta(days=1))["vep-digest"] + + assert current.vep is not None and current.vep.consequence == "stop_gained" + assert historical.vep is not None and historical.vep.consequence == "missense_variant" diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py new file mode 100644 index 000000000..61594e4f0 --- /dev/null +++ b/tests/lib/test_variant_detail.py @@ -0,0 +1,248 @@ +# ruff: noqa: E402 +"""Integration tests for the variant-detail envelope assembly (``lib/variant_detail.py``). + +These pin the composition: the flat assay fields (level, target/reference HGVS, digest, ClinGen id) +off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off +the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications +(primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / +``superseded_by`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +we assert they are wired in and keyed correctly. +""" + +from types import SimpleNamespace + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.variant_detail import get_variant_detail +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from tests.helpers.constants import TEST_MINIMAL_VARIANT, TEST_USER + +# A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this, so it must parse. +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped(), clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + link = MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + session.add(link) + session.commit() + return link + + +def _vep(session, allele, value): + session.add( + VepAlleleConsequence( + allele_id=allele.id, functional_consequence=value, source_version="116", access_date="2026-01-01" + ) + ) + session.commit() + + +def _calibration(session, score_set, *, primary, classifications): + """A calibration with functional classifications; each ``classifications`` entry is + ``(functional, [variants...])`` and its m2m membership is set to those variants.""" + user = session.query(User).filter(User.username == TEST_USER["username"]).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + title=f"cal-{'primary' if primary else 'secondary'}", + primary=primary, + private=False, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + for functional, variants in classifications: + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, label=functional, functional_classification=functional + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +@pytest.mark.integration +def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): + """Mode 1 (coding measured): flat assay fields off the record + authoritative allele, spec-pure + Cat-VRS in projection mode, and a digest-keyed annotation map covering the linked alleles.""" + score_set = setup_lib_db_with_score_set + variant = _variant( + session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" + ) + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein") + _link(session, record, measured, is_authoritative=True) + _link(session, record, protein) + _vep(session, measured, "missense_variant") + + detail = get_variant_detail(session, variant) + + assert detail.urn == variant.urn + assert detail.scores == {"score": -2.3} + assert detail.assay_level == "cdna" + assert detail.target_hgvs == "c.1216G>A" # submitted, target frame + assert detail.reference_hgvs == "NM_000546.6:c.1216G>A" # mapped, reference frame + assert detail.assay_level_digest == "cdna-digest" + assert detail.clingen_allele_id == "CA123" + assert detail.mode == "projection" + assert detail.molecular_representation is not None + assert detail.molecular_representation["type"] == "CategoricalVariant" + # The protein member is a translation of the measured coding allele (member -> defining). + assert detail.member_relations == {"prot-digest": "translation_of"} + # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "prot-digest"} + assert detail.annotations["cdna-digest"].vep is not None + assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" + assert detail.is_current is True + assert detail.superseded_by is None + + +@pytest.mark.integration +def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, and + the nt member `encodes` the defining protein allele.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") + record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") + measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") + coding = _allele(session, "cdna-digest", level="cdna") + _link(session, record, measured, is_authoritative=True) + _link(session, record, coding) + + detail = get_variant_detail(session, variant) + + assert detail.assay_level == "protein" + assert detail.target_hgvs == "p.Ala406Thr" # protein assay -> submitted protein string + assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" + assert detail.assay_level_digest == "prot-digest" + assert detail.mode == "reverse_translation" + assert detail.member_relations == {"cdna-digest": "encodes"} + + +@pytest.mark.integration +def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_score_set): + """No live record: flat mapped fields and Cat-VRS are null, the annotation map is empty, and the + variant reads as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.5}}, hgvs_nt="c.1A>T") + + detail = get_variant_detail(session, variant) + + assert detail.scores == {"score": 0.5} + assert detail.assay_level is None + assert detail.target_hgvs == "c.1A>T" # submitted still surfaces (no assay level -> nucleotide) + assert detail.reference_hgvs is None + assert detail.assay_level_digest is None + assert detail.clingen_allele_id is None + assert detail.molecular_representation is None + assert detail.mode is None + assert detail.member_relations == {} + assert detail.annotations == {} + assert detail.is_current is True + + +@pytest.mark.integration +def test_classifications_are_per_calibration_primary_first(session, setup_lib_db_with_score_set): + """A variant carries one classification per calibration that classifies it; the primary sorts first.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + + detail = get_variant_detail(session, variant) + + assert [(c.primary, c.classification.functional_classification.value) for c in detail.classifications] == [ + (True, "abnormal"), + (False, "normal"), + ] + + +@pytest.mark.integration +def test_visible_calibration_ids_filters_classifications(session, setup_lib_db_with_score_set): + """Only classifications from calibrations the caller resolved as visible are returned.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, data={"score_data": {"score": 0.8}}) + visible = _calibration(session, score_set, primary=True, classifications=[("abnormal", [variant])]) + _calibration(session, score_set, primary=False, classifications=[("normal", [variant])]) + + detail = get_variant_detail(session, variant, visible_calibration_ids={visible.id}) + + assert [c.calibration_id for c in detail.classifications] == [visible.id] + + +@pytest.mark.integration +def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set): + """When a superseding version is passed in, the variant reports it rather than reading as current.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1) + newer = SimpleNamespace(urn="urn:mavedb:00000001-a-2") # only .urn is read + + detail = get_variant_detail(session, variant, superseding_score_set=newer) + + assert detail.is_current is False + assert detail.superseded_by == "urn:mavedb:00000001-a-2" diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..68b48cecc --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,172 @@ +# ruff: noqa: E402 +"""Router tests for the assayed variant-detail endpoint (``GET /variants/{urn}``). + +Exercise the HTTP surface end to end: the two-tier envelope serializes and validates, the ``as_of`` +content-time is echoed in the ``X-As-Of`` header, a superseded variant self-describes rather than +reading as current, and the READ gate holds for private score sets. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.models.vep_allele_consequence import VepAlleleConsequence +from mavedb.view_models.variant_detail import VariantDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + """A spec-valid post_mapped VRS Allele — the Cat-VRS builder hydrates this on the detail path.""" + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed_mapping(session, variant_urn): + """Give a variant a live coding-measured mapping record whose authoritative allele carries a + digest + ClinGen id + a live VEP consequence, plus a protein member — enough to build Cat-VRS.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped=_post_mapped(), clingen_allele_id="CA123") + protein = Allele(vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped()) + session.add_all([measured, protein]) + session.commit() + + session.add_all( + [ + MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + VepAlleleConsequence( + allele_id=measured.id, + functional_consequence="missense_variant", + source_version="116", + access_date="2026-01-01", + ), + ] + ) + session.commit() + + +def test_get_variant_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_mapping(session, urn) + + response = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}") + + assert response.status_code == 200 + body = response.json() + VariantDetail.model_validate(body) + + assert body["urn"] == urn + assert body["assayLevel"] == "cdna" + assert body["referenceHgvs"] == "NM_000546.6:c.1216G>A" + assert body["assayLevelDigest"] == "cdna-digest" + assert body["clingenAlleleId"] == "CA123" + assert body["mode"] == "projection" + # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). + assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + assert body["memberRelations"] == {"prot-digest": "translation_of"} + assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" + assert body["isCurrent"] is True + assert "supersededBy" not in body # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_get_variant_detail_unknown_urn_is_404(client, setup_router_db): + response = client.get("/api/v1/variants/urn:mavedb:00000000-a-1#1") + assert response.status_code == 404 + + +def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + + historical = client.get(f"/api/v1/variants/{urn.replace(chr(35), '%23')}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert historical.status_code == 200 + assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + # Before any mapping existed, the molecular layer is empty. + assert "molecularRepresentation" not in historical.json() + + +def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): + """A variant on a superseded score set is still served, but flags its version standing.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # Link newer as the version that replaces older (replaces_id / superseding relationship). + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + + response = client.get(f"/api/v1/variants/{older['urn']}%231") + + assert response.status_code == 200 + body = response.json() + assert body["isCurrent"] is False + assert body["supersededBy"] == newer["urn"] + + +def test_get_variant_detail_anonymous_cannot_read_private( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/variants/{score_set['urn']}%231") + + assert response.status_code == 404 From 3016dc90154a01a149d7c24b7a8042f688124401 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 11:46:57 -0700 Subject: [PATCH 59/93] refactor(clinical-controls): serve from allele-link substrate with as_of Rework the score-set clinical-controls routes off the legacy MappedVariant.clinical_controls association onto the new-model substrate (ClinvarAlleleLink -> Allele -> MappingRecordAllele -> MappingRecord), matching how the annotation pipeline now writes links. Adds ?as_of= (X-As-Of echo) to reconstruct the allele->ClinVar link state at an instant, and sorts control options newest-version-first. Test infra: add the reusable new-substrate seeder tests/helpers/util/annotation.py (seed_mapping_record + AlleleSpec building the MappingRecord/Allele/link graph), migrate the clinical-controls fixtures (link_clinical_controls_to_alleles) and the lean-view seeder onto it, and retire ClinvarAlleleLinks (not MappedVariant.current) to exercise the live-link filter. --- src/mavedb/routers/score_sets.py | 114 ++++++++++++++++++++++------ tests/helpers/util/annotation.py | 123 +++++++++++++++++++++++++++++++ tests/helpers/util/score_set.py | 32 ++++---- tests/routers/test_score_set.py | 74 +++++++------------ 4 files changed, 260 insertions(+), 83 deletions(-) create mode 100644 tests/helpers/util/annotation.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 34ef50df2..9d150fed0 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -17,8 +17,8 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from pydantic import ValidationError from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound, IntegrityError -from sqlalchemy.orm import Session, contains_eager +from sqlalchemy.exc import IntegrityError, MultipleResultsFound +from sqlalchemy.orm import Session from mavedb import deps from mavedb.data_providers.services import CSV_UPLOAD_S3_BUCKET_NAME, s3_client @@ -74,13 +74,17 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -2448,18 +2452,27 @@ async def publish_score_set( async def get_clinical_controls_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. _db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> Sequence[ClinvarControl]: +) -> list[clinical_control.ClinicalControlWithMappedVariants]: """ Fetch relevant clinical controls for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_controls", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version @@ -2474,25 +2487,38 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) clinical_controls_query = ( - select(ClinvarControl) - .join(ClinvarControl.mapped_variants) - .join(MappedVariant.variant) - .options(contains_eager(ClinvarControl.mapped_variants).contains_eager(MappedVariant.variant)) - .filter(MappedVariant.current.is_(True)) - .filter(Variant.score_set_id == item.id) + select(ClinvarControl, Variant.urn.label("variant_urn")) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == item.id) + .distinct() ) if db_name is not None: save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_name == db_name) + clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_name == db_name) if db_version is not None: save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.filter(ClinvarControl.db_version == db_version) + clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_version == db_version) - clinical_controls: Sequence[ClinvarControl] = _db.scalars(clinical_controls_query).unique().all() + rows = _db.execute(clinical_controls_query).tuples().all() - if not clinical_controls: + controls: dict[int, ClinvarControl] = {} + urns_by_control: dict[int, list[str]] = {} + for ctrl, variant_urn in rows: + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + urns_by_control[ctrl.id] = [] + urns_by_control[ctrl.id].append(variant_urn) + + if not controls: logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2502,9 +2528,25 @@ async def get_clinical_controls_for_score_set( detail=f"No clinical control variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(clinical_controls)}) + save_to_logging_context({"resource_count": len(controls)}) - return clinical_controls + return [ + clinical_control.ClinicalControlWithMappedVariants.model_validate( + { + "id": ctrl.id, + "db_identifier": ctrl.db_identifier, + "gene_symbol": ctrl.gene_symbol, + "clinical_significance": ctrl.clinical_significance, + "clinical_review_status": ctrl.clinical_review_status, + "db_version": ctrl.db_version, + "db_name": ctrl.db_name, + "modification_date": ctrl.modification_date, + "creation_date": ctrl.creation_date, + "mapped_variants": [{"variant_urn": v_urn} for v_urn in urns_by_control[ctrl.id]], + } + ) + for ctrl in controls.values() + ] @router.get( @@ -2518,6 +2560,13 @@ async def get_clinical_controls_for_score_set( async def get_clinical_controls_options_for_score_set( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → ClinVar link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), # We'd prefer to reserve `db` as a query parameter. db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), @@ -2525,7 +2574,13 @@ async def get_clinical_controls_options_for_score_set( """ Fetch clinical control options for a given score set. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "clinical_control_options"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "clinical_control_options", + "as_of": as_of, + } + ) item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2539,13 +2594,19 @@ async def get_clinical_controls_options_for_score_set( clinical_controls_query = ( select(ClinvarControl.db_name, ClinvarControl.db_version) - .join(MappedVariant, ClinvarControl.mapped_variants) - .join(Variant) - .where(MappedVariant.current.is_(True)) + .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) .where(Variant.score_set_id == item.id) + .distinct() ) - clinical_controls_for_item = db.execute(clinical_controls_query).unique() + clinical_controls_for_item = db.execute(clinical_controls_query) # NOTE: We return options even for pairwise groupings which may have no associated mapped variants # and 404 when ultimately requested together. @@ -2563,8 +2624,15 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) + def _version_sort_key(v: str) -> tuple[int, int]: + try: + month, year = v.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) + return [ - dict(zip(("db_name", "available_versions"), (db_name, db_versions))) + {"db_name": db_name, "available_versions": sorted(db_versions, key=_version_sort_key, reverse=True)} for db_name, db_versions in clinical_control_options.items() ] diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py new file mode 100644 index 000000000..7e80a7c73 --- /dev/null +++ b/tests/helpers/util/annotation.py @@ -0,0 +1,123 @@ +"""Seeding helpers for the new-model annotation substrate. + +The serving layer (the lean whole-set view, the ``/variants/{urn}`` detail envelope, allele +annotations, and the score-set clinical-controls routes) reads a variant's mapped alleles and +their annotations through the ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` graph and +the per-source link tables (``ClinvarAlleleLink`` / ``GnomadAlleleLink`` / ``VepAlleleConsequence``) +— not the legacy ``MappedVariant`` association tables. These helpers build that graph in tests so a +score set's variants carry alleles and annotations the way the mapping/annotation pipeline would. + +``seed_mapping_record`` is the single reusable builder; higher-level helpers (e.g. +``link_clinical_controls_to_alleles``) compose it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional, Sequence, Union + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + + +@dataclass +class AlleleSpec: + """One allele on a mapping record, plus the annotations that hang off it. + + Digests are the allele's identity in the serving layer, so they must be distinct within a test. + ``clinvar_control_ids`` / ``gnomad_variant_ids`` reference rows seeded elsewhere (the router + fixtures seed ClinVar controls with ids 1 and 2); each id becomes a live link on this allele. + """ + + digest: str + level: str = "cdna" + is_authoritative: bool = False + clingen_allele_id: Optional[str] = None + hgvs_c: Optional[str] = None + hgvs_p: Optional[str] = None + hgvs_g: Optional[str] = None + post_mapped: Optional[dict] = None + vep_consequence: Optional[str] = None + clinvar_control_ids: Sequence[int] = field(default_factory=tuple) + gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) + + +def seed_mapping_record( + session: Session, + variant: Union[Variant, str], + *, + alleles: Sequence[AlleleSpec], + assay_level: str = "cdna", + hgvs_assay_level: Optional[str] = None, + mapping_api_version: str = "test.0.0", + valid_from: Optional[datetime] = None, +) -> MappingRecord: + """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. + + ``variant`` may be a ``Variant`` instance or its URN. When ``valid_from`` is set it stamps the + record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place + the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row + is live). Returns the created ``MappingRecord``. + """ + if isinstance(variant, str): + resolved = session.scalar(select(Variant).where(Variant.urn == variant)) + assert resolved is not None, f"variant with URN '{variant}' not found" + variant = resolved + + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version=mapping_api_version, + ) + if valid_from is not None: + record.valid_from = valid_from + session.add(record) + session.commit() + + for spec in alleles: + allele = Allele( + vrs_digest=spec.digest, + level=spec.level, + clingen_allele_id=spec.clingen_allele_id, + hgvs_c=spec.hgvs_c, + hgvs_p=spec.hgvs_p, + hgvs_g=spec.hgvs_g, + post_mapped=spec.post_mapped if spec.post_mapped is not None else {"type": "Allele"}, + ) + session.add(allele) + session.commit() + + links: list[Any] = [ + MappingRecordAllele( + mapping_record_id=record.id, allele_id=allele.id, is_authoritative=spec.is_authoritative + ) + ] + if spec.vep_consequence is not None: + links.append( + VepAlleleConsequence( + allele_id=allele.id, + functional_consequence=spec.vep_consequence, + source_version="116", + access_date="2026-01-01", + ) + ) + links.extend(ClinvarAlleleLink(allele_id=allele.id, clinvar_control_id=cid) for cid in spec.clinvar_control_ids) + links.extend(GnomadAlleleLink(allele_id=allele.id, gnomad_variant_id=gid) for gid in spec.gnomad_variant_ids) + + if valid_from is not None: + for link in links: + link.valid_from = valid_from + session.add_all(links) + + session.commit() + return record diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 83ebcbca8..067c627fe 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -24,6 +24,7 @@ TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, ) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.variant import mock_worker_variant_insertion @@ -204,26 +205,31 @@ def create_acc_score_set_with_variants( return score_set -def link_clinical_controls_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) +def link_clinical_controls_to_alleles(db, score_set): + """Seed the new-model annotation graph for a score set's first two variants and link the two + seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. + + The first variant's allele gets the ClinVar control (id 1), the second gets the generic control + (id 2) — mirroring the two-control shape the clinical-controls routes are asserted against. + """ + variants = db.scalars( + select(VariantDbModel) .join(ScoreSetDbModel) .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) ).all() - # The first mapped variant gets the clinvar control, the second gets the generic control. - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) + seed_mapping_record( + db, + variants[0], + alleles=[AlleleSpec(digest="clinical-control-allele-0", is_authoritative=True, clinvar_control_ids=[1])], ) - mapped_variants[1].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 2)) + seed_mapping_record( + db, + variants[1], + alleles=[AlleleSpec(digest="clinical-control-allele-1", is_authoritative=True, clinvar_control_ids=[2])], ) - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() - def link_clinvar_control_to_mapped_variant(db, score_set): """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 055d6bbb2..30e5af017 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4,7 +4,7 @@ import json import re from copy import deepcopy -from datetime import date +from datetime import date, datetime, timezone from io import StringIO from unittest.mock import patch @@ -24,13 +24,9 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline -from mavedb.models.allele import Allele -from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel -from mavedb.models.mapping_record import MappingRecord -from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel -from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.view_models.lean_variant import LeanVariant from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate @@ -80,11 +76,12 @@ create_publish_and_promote_score_calibration, create_test_score_calibration_in_score_set_via_client, ) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.score_set import ( create_seq_score_set, create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, - link_clinical_controls_to_mapped_variants, + link_clinical_controls_to_alleles, link_clinvar_control_to_mapped_variant, link_gnomad_variants_to_mapped_variants, publish_score_set, @@ -3862,7 +3859,7 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") assert response.status_code == 200 @@ -3890,7 +3887,7 @@ def test_can_fetch_current_clinical_controls_for_score_set_with_parameters( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) query_string = "?" for param, accessor in parameters: @@ -3915,7 +3912,7 @@ def test_cannot_fetch_clinical_controls_for_nonexistent_score_set( score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/clinical-controls") @@ -3950,7 +3947,7 @@ def test_can_fetch_current_clinical_control_options_for_score_set( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") assert response.status_code == 200 @@ -3975,17 +3972,14 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s score_set = create_seq_score_set_with_mapped_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinical_controls_to_mapped_variants(session, score_set) + link_clinical_controls_to_alleles(session, score_set) - # Mark all mapped variants as non-current to simulate stale mapping data. - mapped_variants = session.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - for mv in mapped_variants: - mv.current = False + # The options query only surfaces live allele → ClinVar links; retiring them (as a re-map would, + # by stamping valid_to) must leave no current controls and 404. Test-isolated DB, so these are + # the only links present. + links = session.scalars(select(ClinvarAlleleLink).where(ClinvarAlleleLink.valid_to.is_(None))).all() + for link in links: + link.valid_to = datetime.now(timezone.utc) session.commit() response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls/options") @@ -4569,36 +4563,22 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( def _seed_lean_mapping(session, variant_urn): """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" - variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) - record = MappingRecord( - variant_id=variant.id, + seed_mapping_record( + session, + variant_urn, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A", - mapping_api_version="test.0.0", - ) - session.add(record) - session.commit() - - measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped={"type": "Allele"}, clingen_allele_id="CA123") - protein = Allele( - vrs_digest="prot-digest", level="protein", post_mapped={"type": "Allele"}, hgvs_p="NP_000537.3:p.Ala406Thr" - ) - session.add_all([measured, protein]) - session.commit() - - session.add_all( - [ - MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), - MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), - VepAlleleConsequence( - allele_id=measured.id, - functional_consequence="missense_variant", - source_version="116", - access_date="2026-01-01", + alleles=[ + AlleleSpec( + digest="cdna-digest", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + vep_consequence="missense_variant", ), - ] + AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), + ], ) - session.commit() def test_get_lean_variants(client, session, data_provider, data_files, setup_router_db): From a5918d19786ba46c9ed7623e0212790776d31b94 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:10:46 -0700 Subject: [PATCH 60/93] refactor(clinical-controls): serve list and options from a shared lib module The list and options handlers each carried the same ClinvarAlleleLink -> Allele -> MappingRecordAllele -> MappingRecord -> Variant join chain inline, plus an ad-hoc MM_YYYY version sort. Extract the traversal, grouping, and sort into lib/clinical_controls.py so the two endpoints share one definition and cannot drift, matching how the rest of the redesign pushes query logic into lib/. Drop the now-unused model imports and add a unit test pinning the version-sort contract the UI parses in parallel. --- src/mavedb/lib/clinical_controls.py | 105 ++++++++++++++++++++++++++++ src/mavedb/routers/score_sets.py | 82 ++++------------------ tests/lib/test_clinical_controls.py | 32 +++++++++ 3 files changed, 149 insertions(+), 70 deletions(-) create mode 100644 src/mavedb/lib/clinical_controls.py create mode 100644 tests/lib/test_clinical_controls.py diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py new file mode 100644 index 000000000..5aa10b1ce --- /dev/null +++ b/src/mavedb/lib/clinical_controls.py @@ -0,0 +1,105 @@ +"""Live ClinVar-control lookups for a score set, over the allele-link substrate. + +A score set's clinical controls are the ClinVar assertions whose allele is live-linked to one of the +score set's variants, reached by walking ``ClinvarAlleleLink → Allele → MappingRecordAllele → +MappingRecord → Variant`` with every ``ValidTime`` hop constrained to the same instant (``as_of``, +defaulting to currently-live). Both serving endpoints — the controls list +(``GET /score-sets/{urn}/clinical-controls``) and its options facet (``.../options``) — walk that same +chain, so it lives here once: the two cannot drift, and the ``as_of`` semantics are defined in a single +place. +""" + +from datetime import datetime +from typing import Any, Optional, TypeVar + +from sqlalchemy import Select, select +from sqlalchemy.orm import Session + +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant + +SelectT = TypeVar("SelectT", bound=Select[Any]) + + +def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: + """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the + score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the + chain fans a control out across each variant it links to.""" + return ( + stmt.join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) + .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + + +def get_clinical_controls_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_name: Optional[str] = None, + db_version: Optional[str] = None, +) -> list[tuple[ClinvarControl, list[str]]]: + """The live ClinVar controls for a score set, each paired with the URNs of the score-set variants + that link to it. Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). + Controls come back in first-seen order; each control's variant URNs preserve query order.""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl, Variant.urn.label("variant_urn")), score_set_id, as_of + ) + if db_name is not None: + stmt = stmt.where(ClinvarControl.db_name == db_name) + if db_version is not None: + stmt = stmt.where(ClinvarControl.db_version == db_version) + + controls: dict[int, ClinvarControl] = {} + urns_by_control: dict[int, list[str]] = {} + for ctrl, variant_urn in db.execute(stmt).tuples(): + # A persisted variant reached through the join always carries a URN (unique, set at creation); + # narrow the nullable column so the URN list matches the view model's required `str`. + if variant_urn is None: + continue + if ctrl.id not in controls: + controls[ctrl.id] = ctrl + urns_by_control[ctrl.id] = [] + urns_by_control[ctrl.id].append(variant_urn) + + return [(ctrl, urns_by_control[ctrl.id]) for ctrl in controls.values()] + + +def get_clinical_control_options( + db: Session, score_set_id: int, *, as_of: Optional[datetime] = None +) -> list[tuple[str, list[str]]]: + """The options facet: each control DB live-linked to the score set, paired with its available + versions newest-first. Options are returned even for pairwise groupings that may have no variants + when ultimately requested together (the list endpoint 404s that case).""" + stmt = _scope_to_live_score_set_controls( + select(ClinvarControl.db_name, ClinvarControl.db_version), score_set_id, as_of + ) + options: dict[str, list[str]] = {} + for db_name, db_version in db.execute(stmt): + options.setdefault(db_name, []).append(db_version) + + return [ + (db_name, sorted(versions, key=clinvar_version_sort_key, reverse=True)) for db_name, versions in options.items() + ] + + +def clinvar_version_sort_key(version: str) -> tuple[int, int]: + """Sort key for a ClinVar release string in ``MM_YYYY`` form, ordering by ``(year, month)``. + Unparseable versions sort to the bottom as ``(0, 0)`` rather than raising.""" + try: + month, year = version.split("_") + return int(year), int(month) + except (ValueError, AttributeError): + return (0, 0) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 9d150fed0..413d64a07 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -47,6 +47,7 @@ logging_context, save_to_logging_context, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -74,17 +75,12 @@ generate_score_set_urn, ) from mavedb.lib.workflow.pipeline_factory import PipelineFactory -from mavedb.models.allele import Allele -from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.mapping_record import MappingRecord -from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -2475,6 +2471,10 @@ async def get_clinical_controls_for_score_set( # Rename user facing kwargs for consistency with code base naming conventions. db_name = db db_version = version + if db_name is not None: + save_to_logging_context({"db_name": db_name}) + if db_version is not None: + save_to_logging_context({"db_version": db_version}) item: Optional[ScoreSet] = _db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() if not item: @@ -2486,38 +2486,10 @@ async def get_clinical_controls_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinvarControl, Variant.urn.label("variant_urn")) - .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) - .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) - .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) - .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) - .join(Variant, Variant.id == MappingRecord.variant_id) - .where(ClinvarAlleleLink.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(MappingRecord.live_at(as_of)) - .where(Variant.score_set_id == item.id) - .distinct() + controls = get_clinical_controls_with_variant_urns( + _db, item.id, as_of=as_of, db_name=db_name, db_version=db_version ) - if db_name is not None: - save_to_logging_context({"db_name": db_name}) - clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_name == db_name) - - if db_version is not None: - save_to_logging_context({"db_version": db_version}) - clinical_controls_query = clinical_controls_query.where(ClinvarControl.db_version == db_version) - - rows = _db.execute(clinical_controls_query).tuples().all() - - controls: dict[int, ClinvarControl] = {} - urns_by_control: dict[int, list[str]] = {} - for ctrl, variant_urn in rows: - if ctrl.id not in controls: - controls[ctrl.id] = ctrl - urns_by_control[ctrl.id] = [] - urns_by_control[ctrl.id].append(variant_urn) - if not controls: logger.info( msg="No clinical control variants matching the provided filters are associated with the requested score set.", @@ -2542,10 +2514,10 @@ async def get_clinical_controls_for_score_set( "db_name": ctrl.db_name, "modification_date": ctrl.modification_date, "creation_date": ctrl.creation_date, - "mapped_variants": [{"variant_urn": v_urn} for v_urn in urns_by_control[ctrl.id]], + "mapped_variants": [{"variant_urn": v_urn} for v_urn in variant_urns], } ) - for ctrl in controls.values() + for ctrl, variant_urns in controls ] @@ -2592,29 +2564,9 @@ async def get_clinical_controls_options_for_score_set( assert_permission(user_data, item, Action.READ) - clinical_controls_query = ( - select(ClinvarControl.db_name, ClinvarControl.db_version) - .join(ClinvarAlleleLink, ClinvarAlleleLink.clinvar_control_id == ClinvarControl.id) - .join(Allele, Allele.id == ClinvarAlleleLink.allele_id) - .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) - .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) - .join(Variant, Variant.id == MappingRecord.variant_id) - .where(ClinvarAlleleLink.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(MappingRecord.live_at(as_of)) - .where(Variant.score_set_id == item.id) - .distinct() - ) - - clinical_controls_for_item = db.execute(clinical_controls_query) + options = get_clinical_control_options(db, item.id, as_of=as_of) - # NOTE: We return options even for pairwise groupings which may have no associated mapped variants - # and 404 when ultimately requested together. - clinical_control_options: dict[str, list[str]] = {} - for db_name, db_version in clinical_controls_for_item: - clinical_control_options.setdefault(db_name, []).append(db_version) - - if not clinical_control_options: + if not options: logger.info( msg="Failed to fetch clinical control options for score set; No clinical control variants are associated with this score set.", extra=logging_context(), @@ -2624,17 +2576,7 @@ async def get_clinical_controls_options_for_score_set( detail=f"no clinical control variants associated with score set URN {urn} were found", ) - def _version_sort_key(v: str) -> tuple[int, int]: - try: - month, year = v.split("_") - return int(year), int(month) - except (ValueError, AttributeError): - return (0, 0) - - return [ - {"db_name": db_name, "available_versions": sorted(db_versions, key=_version_sort_key, reverse=True)} - for db_name, db_versions in clinical_control_options.items() - ] + return [{"db_name": db_name, "available_versions": available_versions} for db_name, available_versions in options] @router.get( diff --git a/tests/lib/test_clinical_controls.py b/tests/lib/test_clinical_controls.py new file mode 100644 index 000000000..194ff68bd --- /dev/null +++ b/tests/lib/test_clinical_controls.py @@ -0,0 +1,32 @@ +"""Unit tests for the clinical-controls lib helpers (``lib/clinical_controls.py``). + +The live-link query paths are exercised end-to-end by the score-set router tests; these pin the pure +``clinvar_version_sort_key``, which is the ``MM_YYYY`` release-ordering contract the UI parses in +parallel (``lib/clinical-controls.ts``) and so must stay stable. +""" + +from mavedb.lib.clinical_controls import clinvar_version_sort_key + + +def test_version_sort_key_orders_by_year_then_month(): + versions = ["01_2024", "12_2023", "06_2024", "03_2025"] + assert sorted(versions, key=clinvar_version_sort_key, reverse=True) == [ + "03_2025", + "06_2024", + "01_2024", + "12_2023", + ] + + +def test_version_sort_key_parses_month_and_year(): + # (year, month) — a later month within the same year sorts higher. + assert clinvar_version_sort_key("06_2024") == (2024, 6) + assert clinvar_version_sort_key("11_2024") > clinvar_version_sort_key("06_2024") + + +def test_version_sort_key_unparseable_sorts_to_bottom(): + # Malformed versions collapse to (0, 0) rather than raising, so they sink under any real release. + assert clinvar_version_sort_key("garbage") == (0, 0) + assert clinvar_version_sort_key("2024") == (0, 0) + assert clinvar_version_sort_key("13_20_2024") == (0, 0) + assert clinvar_version_sort_key("01_2020") > clinvar_version_sort_key("garbage") From 3717f5d823f779b0ace865853df17977e069d7f1 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:10:54 -0700 Subject: [PATCH 61/93] refactor(variants): rename supersededBy to supersededByScoreSet on variant detail The field carries the superseding *score set*'s URN, not a variant URN, so the old name read as if a consumer could jump straight to a superseding variant. Supersession is versioned at the score-set level and a newer version may add, drop, or renumber variants, so no stable superseding-variant pointer exists; document that alongside the rename. Also modernize get_variant's fetch to a 2.0-style select (preserving the MultipleResultsFound handling). --- src/mavedb/lib/variant_detail.py | 14 +++++++++----- src/mavedb/routers/variants.py | 4 ++-- src/mavedb/view_models/variant_detail.py | 8 ++++++-- tests/lib/test_variant_detail.py | 6 +++--- tests/routers/test_variant.py | 4 ++-- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 09e48a072..3ea10d000 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -5,8 +5,8 @@ on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version -standing (``is_current`` / ``superseded_by``) so a superseded variant self-describes rather than -reading as current (design §9.2). +standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather +than reading as current (design §9.2). Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is @@ -77,7 +77,11 @@ class VariantDetail: # Version standing — self-descriptive for a superseded variant. is_current: bool - superseded_by: Optional[str] # URN of the superseding variant's score set, if any (and readable) + # URN of the score-set version that supersedes this variant's, if any (and readable). This is a + # *score set* URN, not a variant URN — supersession is versioned at the score-set level, and a newer + # version may add/drop/renumber variants, so there is no stable superseding-variant to point at; + # consumers resolve the current measurement by looking this variant up within that score set. + superseded_by_score_set: Optional[str] def _classifications_for_variant( @@ -126,7 +130,7 @@ def get_variant_detail( ``superseding_score_set`` is the newer version the caller has already resolved for visibility (blanked when the user cannot read it, mirroring ``fetch_score_set_by_urn``); its presence drives - ``is_current``/``superseded_by``. ``visible_calibration_ids`` restricts classifications to + ``is_current``/``superseded_by_score_set``. ``visible_calibration_ids`` restricts classifications to readable calibrations. ``as_of`` reconstructs the molecular layer (Cat-VRS membership + annotations); scores and classifications are as-of-invariant. See the module docstring. """ @@ -176,5 +180,5 @@ def get_variant_detail( member_relations=member_relations, annotations=annotations, is_current=superseding_score_set is None, - superseded_by=superseding_score_set.urn if superseding_score_set is not None else None, + superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 97352a8ec..66f302df6 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -465,12 +465,12 @@ def get_variant( Flat assay-level fields for the common UI case plus the spec-pure GA4GH CategoricalVariant and a digest-keyed annotation map for machine/standard consumers. A superseded variant is served (it is - the citable unit) but self-describes via isCurrent/supersededBy rather than reading as current. + the citable unit) but self-describes via isCurrent/supersededByScoreSet rather than reading as current. """ save_to_logging_context({"requested_resource": urn, "as_of": as_of}) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" try: - variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() except MultipleResultsFound: logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 18a78f844..2c242f756 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -36,7 +36,11 @@ class VariantDetail(BaseModel): is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` - /``supersededBy`` let a superseded variant self-describe. Absent fields are omitted. + /``supersededByScoreSet`` let a superseded variant self-describe: ``supersededByScoreSet`` is the + superseding *score set*'s URN, not a variant URN. Supersession is versioned at the score-set level, + and a newer version may add, drop, or renumber variants — so there is no stable + superseding-*variant* pointer to hand back; a consumer resolves the current measurement by looking + this variant up within that score set. Absent fields are omitted. """ urn: str @@ -57,7 +61,7 @@ class VariantDetail(BaseModel): annotations: dict[str, AlleleAnnotations] = {} is_current: bool - superseded_by: Optional[str] = None + superseded_by_score_set: Optional[str] = None class Config: from_attributes = True diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 61594e4f0..6cea44858 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -5,7 +5,7 @@ off the live record + authoritative allele; the spec-pure Cat-VRS + MaveDB mode/member-relations off the on-the-fly builder; the digest-keyed annotation map; the per-calibration classifications (primary first, filterable by visibility); scores/counts passthrough; and the ``is_current`` / -``superseded_by`` version standing. Cat-VRS/annotation internals are covered in their own suites — here +``superseded_by_score_set`` version standing. Cat-VRS/annotation internals are covered in their own suites — here we assert they are wired in and keyed correctly. """ @@ -159,7 +159,7 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True - assert detail.superseded_by is None + assert detail.superseded_by_score_set is None @pytest.mark.integration @@ -245,4 +245,4 @@ def test_superseded_variant_self_describes(session, setup_lib_db_with_score_set) detail = get_variant_detail(session, variant, superseding_score_set=newer) assert detail.is_current is False - assert detail.superseded_by == "urn:mavedb:00000001-a-2" + assert detail.superseded_by_score_set == "urn:mavedb:00000001-a-2" diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 68b48cecc..365fe9ca7 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -109,7 +109,7 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["memberRelations"] == {"prot-digest": "translation_of"} assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True - assert "supersededBy" not in body # dropped by exclude_none when current + assert "supersededByScoreSet" not in body # dropped by exclude_none when current assert response.headers["X-As-Of"] == "current" @@ -154,7 +154,7 @@ def test_superseded_variant_self_describes(client, session, data_provider, data_ assert response.status_code == 200 body = response.json() assert body["isCurrent"] is False - assert body["supersededBy"] == newer["urn"] + assert body["supersededByScoreSet"] == newer["urn"] def test_get_variant_detail_anonymous_cannot_read_private( From ac8f4052d03fc4b456aaba8884c7f8fabd439d95 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 14:11:03 -0700 Subject: [PATCH 62/93] fix(mapping): enforce one live authoritative allele link per record The lean whole-set view joins the authoritative MappingRecordAllele expecting it 1:1 with the variant. The existing (mapping_record, allele) unique index cannot stop two different alleles both being flagged authoritative for one record, so a second live authoritative link would silently duplicate the variant and double-count its score rather than raise. Add a partial unique index on (mapping_record_id) WHERE is_authoritative AND valid_to IS NULL so the mistake fails loud in the mapping job at write time. Detection query is clean on dev data. --- ...ique_live_authoritative_link_per_record.py | 46 +++++++++++++++++++ src/mavedb/models/mapping_record_allele.py | 14 ++++++ 2 files changed, 60 insertions(+) create mode 100644 alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py diff --git a/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py new file mode 100644 index 000000000..6d7b6c989 --- /dev/null +++ b/alembic/versions/f1a3c7e9b2d5_unique_live_authoritative_link_per_record.py @@ -0,0 +1,46 @@ +"""enforce one live authoritative link per mapping record + +Revision ID: f1a3c7e9b2d5 +Revises: c9b2a4f8e1d3 +Create Date: 2026-07-07 + +Adds a partial unique index enforcing at most one live authoritative link per mapping record. The +existing uq_mapping_record_alleles_live keys on (mapping_record, allele) and so cannot prevent two +different alleles both being flagged is_authoritative for the same record. The serving layer depends +on there being exactly one: the lean whole-set view (lib/score_set_variants) joins the authoritative +link expecting it 1:1 with the variant, so a second live authoritative link would silently duplicate +the variant and double-count its score in the histogram rather than raise. This index moves that +failure to write time in the mapping job, where the bug actually lives. + +Assumes no pre-existing record with two live authoritative links — the invariant is upheld today by +the mapping job's write logic. If this ever runs against data that violates it, the unique index +creation fails and the offending duplicates must be retired first. To check before applying: + + SELECT mapping_record_id, count(*) + FROM mapping_record_alleles + WHERE is_authoritative AND valid_to IS NULL + GROUP BY mapping_record_id HAVING count(*) > 1; +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1a3c7e9b2d5" +down_revision = "c9b2a4f8e1d3" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_alleles", + ["mapping_record_id"], + unique=True, + postgresql_where=sa.text("is_authoritative AND valid_to IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_mapping_record_alleles_live_authoritative", table_name="mapping_record_alleles") diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index cda8bf0f3..7bfbd7885 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -57,4 +57,18 @@ class MappingRecordAllele(ValidTime, Base): unique=True, postgresql_where=text("valid_to IS NULL"), ), + # At most one live *authoritative* link per mapping record. The index above keys on + # (record, allele) and so cannot stop two different alleles both being flagged authoritative + # for one record; this one enforces the "one authoritative (measured) allele per record" + # invariant directly. It is the backstop the serving layer relies on: the lean whole-set view + # (lib/score_set_variants) inner-joins the authoritative link expecting it 1:1 with the + # variant, so a second live authoritative link would silently duplicate the variant (and + # double-count its score) rather than raise. With this index, that mistake fails loud in the + # mapping job at write time instead. + Index( + "uq_mapping_record_alleles_live_authoritative", + "mapping_record_id", + unique=True, + postgresql_where=text("is_authoritative AND valid_to IS NULL"), + ), ) From 7b92d6936fe1551bf20e0a34603efc595dfd49fc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:33:40 -0700 Subject: [PATCH 63/93] refactor(calibrations): extract shared calibration-preference and evidence helpers Calibration selection had diverged across the serving layer (variant_detail ordered by primary/id; the new allele-measurements list used a fuller cascade). Add two pure helpers to lib/score_calibrations: - calibration_preference_key: the UI's default-calibration cascade (primary -> investigator_provided -> non-research-use-only), so the API and UI agree on the default calibration for a score set. - classification_evidence_strength: the (magnitude, direction) evidence primitive (ACMG points -> |ln(oddspath)| -> functional call). Repoint variant_detail's classification ordering onto the shared key so the variant page's first-listed classification matches the measurement card and the UI default. Annotate ScoreCalibration.id/primary as Mapped for the typed helpers. --- src/mavedb/lib/score_calibrations.py | 51 ++++++++++++++++++++++++++ src/mavedb/lib/variant_detail.py | 23 ++++++++---- src/mavedb/models/score_calibration.py | 4 +- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/mavedb/lib/score_calibrations.py b/src/mavedb/lib/score_calibrations.py index f7b84580f..25608df7c 100644 --- a/src/mavedb/lib/score_calibrations.py +++ b/src/mavedb/lib/score_calibrations.py @@ -18,6 +18,7 @@ ) from mavedb.lib.validation.utilities import inf_or_float from mavedb.lib.variants import score_from_variant_data +from mavedb.models.enums.functional_classification import FunctionalClassification from mavedb.models.enums.score_calibration_relation import ScoreCalibrationRelation from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -28,6 +29,56 @@ from mavedb.view_models import score_calibration +def calibration_preference_key(calibration: ScoreCalibration) -> tuple[int, int, int]: + """The UI's default-calibration preference cascade as a sort key (lower sorts first = more preferred): + ``primary``, then ``investigator_provided``, then non-``research_use_only``. Mirrors the UI's + ``activeCalibrationOptions`` default selection, so the API and UI agree on which calibration is "the + default" for a score set. + + Intentionally stops at the cascade tiers and carries **no** evidence strength (a variant-specific + concern) and **no** id — callers append their own deterministic tiebreak (calibration id or urn), and, where + a within-tier strongest-evidence tiebreak is wanted, :func:`classification_evidence_strength`. + """ + return ( + 0 if calibration.primary else 1, + 0 if calibration.investigator_provided else 1, + 0 if not calibration.research_use_only else 1, + ) + + +def classification_evidence_strength( + classification: Optional[ScoreCalibrationFunctionalClassification], +) -> tuple[float, int]: + """``(magnitude, direction)`` of a functional classification's evidence, for ranking. + + ``magnitude`` is the evidence strength (larger = stronger); ``direction`` biases toward pathogenic + (0 = pathogenic, 1 = benign, 2 = unknown). ACMG points are the primary signal (signed: + positive = pathogenic), falling back to ``|ln(oddspaths_ratio)|`` then the functional call + (abnormal → pathogenic, normal → benign) for direction only. + """ + if classification is None: + return (0.0, 2) + + acmg = classification.acmg_classification + if acmg is not None and acmg.points is not None: + points = float(acmg.points) + return (abs(points), 0 if points > 0 else 1 if points < 0 else 2) + + # OddsPath is a ratio centered at 1 (pathogenic >1, benign <1); ln maps it to a symmetric log-odds + # strength around 0 (x and 1/x are equal-and-opposite), comparable across direction and commensurate + # with the already-log-scaled ACMG points. + ratio = classification.oddspaths_ratio + if ratio is not None and ratio > 0: + return (abs(math.log(ratio)), 0 if ratio > 1 else 1 if ratio < 1 else 2) + + if classification.functional_classification == FunctionalClassification.abnormal: + return (0.0, 0) + elif classification.functional_classification == FunctionalClassification.normal: + return (0.0, 1) + + return (0.0, 2) + + def create_functional_classification( db: Session, functional_range_create: Union[ diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 3ea10d000..0331c7d7a 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -25,6 +25,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.annotation_layer import AnnotationLayer from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_calibration import ScoreCalibration @@ -88,31 +89,37 @@ def _classifications_for_variant( db: Session, variant: Variant, *, visible_calibration_ids: Optional[set[int]] ) -> list[VariantClassificationRecord]: """The functional classifications the variant belongs to (membership is materialized on the - calibration→classification→variant m2m). One row per calibration that classifies the variant; - the primary calibration sorts first. ``visible_calibration_ids`` restricts to calibrations the - caller resolved as readable (``None`` = no restriction, for lib-level use).""" + calibration→classification→variant m2m). One row per calibration that classifies the variant, ordered + by the shared calibration preference cascade (``calibration_preference_key`` + id) so the first entry + is the same default the UI and the allele-measurements card surface. ``visible_calibration_ids`` + restricts to calibrations the caller resolved as readable (``None`` = no restriction, for lib-level use).""" statement = ( - select(ScoreCalibrationFunctionalClassification, ScoreCalibration.primary, ScoreCalibration.id) + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) .join( classification_variants, classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, ) .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) .where(classification_variants.c.variant_id == variant.id) - .order_by(ScoreCalibration.primary.desc(), ScoreCalibration.id) ) if visible_calibration_ids is not None: statement = statement.where(ScoreCalibration.id.in_(visible_calibration_ids)) + rows = sorted( + db.execute(statement).tuples().all(), key=lambda row: (*calibration_preference_key(row[1]), row[1].id) + ) return [ - VariantClassificationRecord(calibration_id=calibration_id, primary=primary, classification=classification) - for classification, primary, calibration_id in db.execute(statement) + VariantClassificationRecord( + calibration_id=calibration.id, primary=calibration.primary, classification=classification + ) + for classification, calibration in rows ] def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, - otherwise the nucleotide expression (genomic or coding share ``hgvs_nt``).""" + otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is + never an index column).""" if assay_level == AnnotationLayer.protein.value: return variant.hgvs_pro return variant.hgvs_nt diff --git a/src/mavedb/models/score_calibration.py b/src/mavedb/models/score_calibration.py index 38ce1f286..1941cb863 100644 --- a/src/mavedb/models/score_calibration.py +++ b/src/mavedb/models/score_calibration.py @@ -25,7 +25,7 @@ class ScoreCalibration(Base): __tablename__ = "score_calibrations" # TODO#544: Add a partial unique index to enforce only one primary calibration per score set. - id = Column(Integer, primary_key=True) + id: Mapped[int] = Column(Integer, primary_key=True) urn = Column(String(64), nullable=True, default=generate_calibration_urn, unique=True, index=True) score_set_id = Column(Integer, ForeignKey("scoresets.id"), nullable=False) @@ -33,7 +33,7 @@ class ScoreCalibration(Base): title = Column(String, nullable=False) research_use_only = Column(Boolean, nullable=False, default=False) - primary = Column(Boolean, nullable=False, default=False) + primary: Mapped[bool] = Column(Boolean, nullable=False, default=False) investigator_provided: Mapped[bool] = Column(Boolean, nullable=False, default=False) private = Column(Boolean, nullable=False, default=True) notes = Column(String, nullable=True) From e47791c0d52922e3c9696f3271193b31f71677b8 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:34:12 -0700 Subject: [PATCH 64/93] feat(clingen-alleles): add equivalence-class measurements endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /clingen-alleles/{caid}/measurements — the ClinGen-allele-centric variant page's entrypoint. Resolves a CA/PA to its anchor alleles and lists every measurement whose cross-layer equivalence class (MappingRecordAllele co-membership) touches it, not just the exact allele. Each card is labeled by its assayed level and its relationship to the query (direct / protein_consequence / nucleotide_encoding); the split falls out of which allele anchors the co-membership. Carries the primary readable classification inline, orders direct-first then by strongest pathogenic-biased evidence, and opts into superseded measurements and as_of. Authorization is has_permission in the lib (score-set READ gates inclusion, calibration READ gates the inline classification). --- src/mavedb/lib/allele_measurements.py | 234 ++++++++++++ src/mavedb/routers/clingen_alleles.py | 85 +++++ src/mavedb/server_main.py | 3 + src/mavedb/view_models/allele_measurement.py | 41 ++ tests/lib/test_allele_measurements.py | 371 +++++++++++++++++++ tests/routers/test_clingen_allele.py | 122 ++++++ 6 files changed, 856 insertions(+) create mode 100644 src/mavedb/lib/allele_measurements.py create mode 100644 src/mavedb/routers/clingen_alleles.py create mode 100644 src/mavedb/view_models/allele_measurement.py create mode 100644 tests/lib/test_allele_measurements.py create mode 100644 tests/routers/test_clingen_allele.py diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py new file mode 100644 index 000000000..eabb414f5 --- /dev/null +++ b/src/mavedb/lib/allele_measurements.py @@ -0,0 +1,234 @@ +"""The ClinGen-allele-centric variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id (a nucleotide ``CA`` or a protein ``PA``), list every measurement whose +**cross-layer equivalence class** touches that allele — not just the exact allele. The equivalence +relation is co-membership in a ``MappingRecord``'s allele set (the mapping + reverse-translation graph, +:mod:`lib.alleles`): a ``CA`` anchors on its nucleotide alleles (genomic + coding share the CA), so the +list gathers the nt measurements of that change (**direct**) *and* the protein measurements of its +consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of +that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls +out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein +is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. See +design §7.5. + +Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels +detail of one *selected* measurement. This list is the *cross-record* union — who else measured +something in the equivalence class. + +Authorization uses ``has_permission`` directly (matching ``lib/score_sets.py``): ``user_data`` is checked +per entity — score-set READ gates whether a measurement appears at all (a private score set's measurement +never leaks), and calibration READ gates the inline classification (withheld while the measurement still +shows). The two-gate split is deliberate: a readable measurement can carry an unreadable classification. +``as_of`` reconstructs the molecular layer (which records/links are live) at a past instant; scores are +immutable and calibrations carry no valid-time, so both are as-of-invariant. +""" + +from dataclasses import dataclass +from datetime import date, datetime +from enum import Enum +from typing import Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, aliased, joinedload + +from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength +from mavedb.lib.types.authentication import UserData +from mavedb.lib.variants import variant_score +from mavedb.models.allele import Allele +from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table as classification_variants, +) +from mavedb.models.variant import Variant + + +class MeasurementRelationship(str, Enum): + """The relationship of a measurement to the queried ClinGen allele. ``direct`` = the measurement was + assayed *at* this allele; the other two are the RT-related measurements, named for how they relate + to the query. + """ + + direct = "direct" # the measurement was assayed *at* this allele + protein_consequence = "protein_consequence" # CA query, protein measurement of consequence(N) + nucleotide_encoding = "nucleotide_encoding" # PA query, nt measurement encoding P + + +@dataclass(frozen=True) +class AlleleMeasurement: + """One measurement in the queried allele's equivalence class (transit; serialized by + ``view_models.allele_measurement``). + + ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the + clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried + ClinGen id. ``primary_classification`` is the primary readable functional classification (``None`` + when the score set has none or the caller cannot read the calibration). + """ + + variant_urn: str + score: Optional[float] + assay_level: Optional[str] + relationship: str + assay_level_hgvs: Optional[str] + submitted_hgvs: Optional[str] + score_set_urn: str + score_set_title: str + primary_classification: Optional[ScoreCalibrationFunctionalClassification] + is_current: bool + superseded_by_score_set: Optional[str] + + +def _preferred_classification( + db: Session, variant: Variant, *, user_data: Optional[UserData] +) -> Optional[ScoreCalibrationFunctionalClassification]: + """The variant's preferred *readable* functional classification, or ``None``. + + A variant falls into one classification per calibration (non-overlapping ranges); we surface the one + the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then + ``investigator_provided``, then non-``research_use_only``. Within a tier we take the strongest + evidence (the VA-Spec posture — the strongest calibration is the evidence calibration), then ``id`` + only for determinism. The UI's "any with ranges / any calibration" tail steps collapse here: this + query only surfaces calibrations that already classify the variant. Only calibrations the caller can + read (calibration READ) are considered, so a private calibration is skipped rather than blanking a + readable call. Kept in sync with the UI's `activeCalibrationOptions` default selection. + """ + candidates = [ + (classification, calibration) + for classification, calibration in db.execute( + select(ScoreCalibrationFunctionalClassification, ScoreCalibration) + .join( + classification_variants, + classification_variants.c.functional_classification_id == ScoreCalibrationFunctionalClassification.id, + ) + .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) + .where(classification_variants.c.variant_id == variant.id) + ).all() + if has_permission(user_data, calibration, Action.READ).permitted + ] + if not candidates: + return None + + def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibration]) -> tuple: + classification, calibration = pair + magnitude, _ = classification_evidence_strength(classification) + return ( + *calibration_preference_key(calibration), + -magnitude, + calibration.id, + ) + + return min(candidates, key=preference)[0] + + +def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: + """Sort order: + 1. Direct measurements, then related (protein_consequence / nucleotide_encoding) + 2. Within each, the strongest evidence first (pathogenic wins ties) + 3. Within each, the newest-published first + 4. Within each, the URN for a stable tiebreak + """ + magnitude, direction = classification_evidence_strength(measurement.primary_classification) + return ( + 0 if measurement.relationship == MeasurementRelationship.direct else 1, + 0 if measurement.primary_classification is not None else 1, + -magnitude, + direction, + -(published_date.toordinal()) if published_date is not None else 0, + measurement.variant_urn, + ) + + +def get_allele_measurements( + db: Session, + clingen_allele_id: str, + *, + user_data: Optional[UserData], + include_superseded: bool = False, + as_of: Optional[datetime] = None, +) -> list[AlleleMeasurement]: + """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns + ``[]`` when the id resolves to no allele or no live record. + """ + anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() + if not anchor: + return [] + + anchor_ids = [row.id for row in anchor] + entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + + record_ids = db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.allele_id.in_(anchor_ids)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + if not record_ids: + return [] + + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. + authoritative = aliased(MappingRecordAllele) + rows = ( + db.execute( + select(MappingRecord, Variant, Allele) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join( + authoritative, + and_( + authoritative.mapping_record_id == MappingRecord.id, + authoritative.is_authoritative.is_(True), + authoritative.live_at(as_of), + ), + ) + .join(Allele, Allele.id == authoritative.allele_id) + .where(MappingRecord.id.in_(record_ids)) + .where(MappingRecord.live_at(as_of)) + .options(joinedload(Variant.score_set)) + ) + .tuples() + .all() + ) + + measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + for record, variant, measured_allele in rows: + score_set = variant.score_set + if not has_permission(user_data, score_set, Action.READ).permitted: + continue + + # Don't leak unpermitted resources. + superseding = score_set.superseding_score_set + if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: + superseding = None + + is_current = superseding is None + if not is_current and not include_superseded: + continue + + if measured_allele.clingen_allele_id == clingen_allele_id: + relationship = MeasurementRelationship.direct + elif entry_is_protein: + relationship = MeasurementRelationship.nucleotide_encoding + else: + relationship = MeasurementRelationship.protein_consequence + + assay_level = record.assay_level + measurement = AlleleMeasurement( + variant_urn=variant.urn or "", + score=variant_score(variant), + assay_level=assay_level, + relationship=relationship, + assay_level_hgvs=record.hgvs_assay_level, + submitted_hgvs=variant.hgvs_pro if assay_level == AnnotationLayer.protein.value else variant.hgvs_nt, + score_set_urn=score_set.urn or "", + score_set_title=score_set.title or "", + primary_classification=_preferred_classification(db, variant, user_data=user_data), + is_current=is_current, + superseded_by_score_set=superseding.urn if superseding is not None else None, + ) + measurements.append((measurement, score_set.published_date)) + + measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) + return [measurement for measurement, _ in measurements] diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py new file mode 100644 index 000000000..b3b626c02 --- /dev/null +++ b/src/mavedb/routers/clingen_alleles.py @@ -0,0 +1,85 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Query, Response +from sqlalchemy.orm import Session + +from mavedb import deps +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.authentication import get_current_user +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.lib.types.authentication import UserData +from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX +from mavedb.view_models.allele_measurement import AlleleMeasurement + +TAG_NAME = "ClinGen alleles" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/clingen-alleles", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "The ClinGen-allele-centric variant page: measurements across a variant's equivalence class.", +} + + +@router.get( + "/{clingen_allele_id}/measurements", + status_code=200, + response_model=list[AlleleMeasurement], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="List measurements for a ClinGen allele's equivalence class", +) +def get_clingen_allele_measurements( + *, + clingen_allele_id: str, + response: Response, + include_superseded: bool = Query( + default=False, + description=( + "Include measurements from superseded score-set versions. Default false — superseded " + "measurements are a deliberate power-user / citation path, never surfaced by discovery." + ), + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the equivalence class (which mapping records / allele links are live) as it " + "stood at this instant. ISO 8601, ideally timezone-aware. Scores and classifications are " + "as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +): + """List every measurement whose cross-layer equivalence class touches this ClinGen allele (a ``CA`` or + ``PA``) — the direct measurements assayed at this change plus the reverse-translation-related ones, + each labeled by its assayed level and relationship. This is the ClinGen-allele-centric variant page's + entrypoint. A private score set's measurement is never included; its inline classification is withheld + where the calibration is unreadable while the measurement still shows. + """ + save_to_logging_context( + { + "requested_resource": clingen_allele_id, + "as_of": as_of, + "include_superseded": include_superseded, + } + ) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + return get_allele_measurements( + db, + clingen_allele_id, + user_data=user_data, + include_superseded=include_superseded, + as_of=as_of, + ) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 150714883..e79819999 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -41,6 +41,7 @@ access_keys, alphafold, api_information, + clingen_alleles, collections, controlled_keywords, doi_identifiers, @@ -93,6 +94,7 @@ app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) app.include_router(api_information.router) +app.include_router(clingen_alleles.router) app.include_router(collections.router) app.include_router(controlled_keywords.router) app.include_router(doi_identifiers.router) @@ -241,6 +243,7 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, api_information.metadata, + clingen_alleles.metadata, collections.metadata, controlled_keywords.metadata, doi_identifiers.metadata, diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py new file mode 100644 index 000000000..d61197524 --- /dev/null +++ b/src/mavedb/view_models/allele_measurement.py @@ -0,0 +1,41 @@ +"""Response view model for the variant page's measurements list +(``GET /clingen-alleles/{caid}/measurements``, design §7.5). + +The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass +(``from_attributes`` coerces it directly); aliases camelize per the shared base config. +""" + +from typing import Optional + +from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_calibration import SavedFunctionalClassification + + +class AlleleMeasurement(BaseModel): + """One measurement in the queried ClinGen allele's cross-layer equivalence class. + + ``assayLevel`` is the level at which this measurement was actually assayed (``protein`` / ``cdna`` / + ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. + ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at + this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``primaryClassification`` is the + primary readable functional classification, omitted when absent or gated. ``isCurrent`` / + ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) + self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. + """ + + variant_urn: str + score: Optional[float] = None + assay_level: Optional[str] = None + relationship: MeasurementRelationship + assay_level_hgvs: Optional[str] = None + submitted_hgvs: Optional[str] = None + score_set_urn: str + score_set_title: str + primary_classification: Optional[SavedFunctionalClassification] = None + is_current: bool + superseded_by_score_set: Optional[str] = None + + class Config: + from_attributes = True diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py new file mode 100644 index 000000000..6c49f6713 --- /dev/null +++ b/tests/lib/test_allele_measurements.py @@ -0,0 +1,371 @@ +# ruff: noqa: E402 +"""Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). + +These pin the model that the first pass got wrong: measurements aggregate the **cross-layer equivalence +class** (co-membership in the mapping/RT link graph), not one exact allele, and *which* allele anchors it +determines the direct/related split (a CA anchors on the nt alleles → nt measurements are direct, the +protein consequence is related; a PA anchors on the protein → protein measurements are direct, the nt +encodings related). Also: the narrowness (a sibling nt change is not pulled onto a CA page), the two +authorization gates (score-set READ hides the measurement; calibration READ withholds only the inline +classification) exercised through the real ``has_permission``, superseded opt-in, and the default ordering +(direct-first, strongest evidence, pathogenic-biased). +""" + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.allele_measurements import get_allele_measurements +from mavedb.lib.types.authentication import UserData +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import EXTRA_USER, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, TEST_USER + + +def _user(session, username): + return session.query(User).filter(User.username == username).one() + + +def _user_data(session, username=TEST_USER["username"]): + """A ``UserData`` for a seeded user — the currency ``has_permission`` reads. ``None`` is anonymous.""" + return UserData(user=_user(session, username), active_roles=[]) + + +def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=None): + variant = Variant( + urn=f"{score_set.urn}#{suffix}", + score_set_id=score_set.id, + data=data if data is not None else TEST_MINIMAL_VARIANT["data"], + hgvs_nt=hgvs_nt, + hgvs_pro=hgvs_pro, + creation_date=TEST_MINIMAL_VARIANT["creation_date"], + modification_date=TEST_MINIMAL_VARIANT["modification_date"], + ) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest, *, level="cdna", clingen_allele_id=None): + allele = Allele(vrs_digest=digest, level=level, clingen_allele_id=clingen_allele_id) + session.add(allele) + session.commit() + return allele + + +def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): + record = MappingRecord( + variant_id=variant.id, + assay_level=assay_level, + hgvs_assay_level=hgvs_assay_level, + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + return record + + +def _link(session, record, allele, *, is_authoritative=False): + session.add( + MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) + ) + session.commit() + + +def _calibration( + session, + score_set, + *, + variants, + functional="abnormal", + oddspaths_ratio=None, + primary=True, + private=False, + investigator_provided=False, + research_use_only=False, +): + user = _user(session, TEST_USER["username"]) + calibration = ScoreCalibration( + score_set_id=score_set.id, + title="cal", + primary=primary, + private=private, + investigator_provided=investigator_provided, + research_use_only=research_use_only, + created_by_id=user.id, + modified_by_id=user.id, + ) + session.add(calibration) + session.commit() + fc = ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label=functional, + functional_classification=functional, + oddspaths_ratio=oddspaths_ratio, + ) + fc.variants = variants + session.add(fc) + session.commit() + return calibration + + +def _second_score_set( + session, base, urn, *, owner_username=TEST_USER["username"], private=True, superseded_score_set_id=None +): + """A sibling score set in the same experiment — for cross-score-set, private, and supersession tests. + ``superseded_score_set_id`` set on this (newer) row makes it the superseding version of that older + set (``older.superseding_score_set`` resolves back here).""" + owner = _user(session, owner_username) + scaffold = TEST_SEQ_SCORESET.copy() + scaffold.pop("target_genes", None) + score_set = ScoreSet( + **scaffold, + urn=urn, + experiment_id=base.experiment_id, + licence_id=base.licence_id, + superseded_score_set_id=superseded_score_set_id, + ) + score_set.private = private + score_set.created_by = owner + score_set.modified_by = owner + session.add(score_set) + session.commit() + session.refresh(score_set) + return score_set + + +def _seed_two_level_pair(session, score_set): + """One nt change (CA123) and its protein consequence (PA9), each measured in its own assay, with the + other level riding as a member — the minimal shape that exercises the direct/related asymmetry.""" + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna", hgvs_assay_level="NM_0:c.1A>T") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein", hgvs_assay_level="NP_0:p.Met1Leu") + _link(session, protein_record, prot, is_authoritative=True) + _link(session, protein_record, nt) + + return nt_measured, protein_measured + + +@pytest.mark.integration +def test_ca_entry_aggregates_direct_and_related(session, setup_lib_db_with_score_set): + """A CA page: the nt measurement is direct (assayed at this change); the protein measurement of its + consequence is related, labeled ``protein_consequence``, each carrying its own measured level.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert (by_urn[nt_measured.urn].relationship, by_urn[nt_measured.urn].assay_level) == ("direct", "cdna") + assert (by_urn[protein_measured.urn].relationship, by_urn[protein_measured.urn].assay_level) == ( + "protein_consequence", + "protein", + ) + # Assay HGVS pair rides through (mapped reference + submitted target). + assert by_urn[nt_measured.urn].assay_level_hgvs == "NM_0:c.1A>T" + assert by_urn[nt_measured.urn].submitted_hgvs == "c.1A>T" + assert by_urn[protein_measured.urn].submitted_hgvs == "p.Met1Leu" + + +@pytest.mark.integration +def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set): + """The same data anchored on the PA: the protein measurement is now direct and the nt measurement is + the related ``nucleotide_encoding`` — the asymmetry is purely which allele anchors the co-membership.""" + score_set = setup_lib_db_with_score_set + nt_measured, protein_measured = _seed_two_level_pair(session, score_set) + + result = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "direct" + assert by_urn[nt_measured.urn].relationship == "nucleotide_encoding" + + +@pytest.mark.integration +def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set): + """A sibling nt change that encodes the *same* protein is not pulled onto a CA page (its record links + the protein but not this nt allele) — but both nt changes show on the protein's PA page as encodings.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, prot) + + ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) + assert {m.variant_urn for m in ca_page} == {a.urn} + + pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) + assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} + assert all(m.relationship == "nucleotide_encoding" for m in pa_page) + + +@pytest.mark.integration +def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): + """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, + even though it links the same (public, content-addressed) allele.""" + score_set = setup_lib_db_with_score_set # private, owned by TEST_USER + other = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", owner_username=EXTRA_USER["username"]) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + + visible = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, visible, assay_level="cdna"), nt, is_authoritative=True) + hidden = _variant(session, other, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, hidden, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert {m.variant_urn for m in result} == {visible.urn} + + +@pytest.mark.integration +def test_calibration_gate_withholds_classification(session, setup_lib_db_with_score_set): + """Calibration READ gates only the inline classification: on a public score set with a private + calibration, an anonymous caller still sees the measurement but its classification is withheld, while + the owner sees both.""" + score_set = setup_lib_db_with_score_set + public = _second_score_set(session, score_set, "urn:mavedb:00000002-a-1", private=False) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, public, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, public, variants=[variant], functional="abnormal", private=True) + + anonymous = get_allele_measurements(session, "CA123", user_data=None) + assert len(anonymous) == 1 + assert anonymous[0].primary_classification is None + + owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert owner[0].primary_classification is not None + assert owner[0].primary_classification.functional_classification.value == "abnormal" + + +@pytest.mark.integration +def test_primary_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): + """With no primary calibration, the preference cascade dominates evidence strength: an + investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) # plain, non-RUO + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0, investigator_provided=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. + assert result[0].primary_classification.oddspaths_ratio == 2.0 + + +@pytest.mark.integration +def test_primary_classification_strongest_within_tier(session, setup_lib_db_with_score_set): + """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=2.0) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].primary_classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_superseded_excluded_by_default_included_opt_in(session, setup_lib_db_with_score_set): + """Superseded measurements are current-tail-only by default; ``include_superseded`` opts in and they + self-describe (``is_current`` false + the superseding score set's URN).""" + score_set = setup_lib_db_with_score_set + _second_score_set(session, score_set, "urn:mavedb:00000001-a-2", superseded_score_set_id=score_set.id) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + assert get_allele_measurements(session, "CA123", user_data=_user_data(session)) == [] + + opted = get_allele_measurements(session, "CA123", user_data=_user_data(session), include_superseded=True) + assert len(opted) == 1 + assert opted[0].is_current is False + assert opted[0].superseded_by_score_set == "urn:mavedb:00000001-a-2" + + +@pytest.mark.integration +def test_unreadable_superseding_reads_as_current(session, setup_lib_db_with_score_set): + """A superseding version the caller cannot read is not leaked — the measurement reads as current and + is included by default.""" + score_set = setup_lib_db_with_score_set # owned by TEST_USER + _second_score_set( + session, + score_set, + "urn:mavedb:00000001-a-2", + owner_username=EXTRA_USER["username"], + superseded_score_set_id=score_set.id, + ) + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # TEST_USER + assert len(result) == 1 + assert result[0].is_current is True + assert result[0].superseded_by_score_set is None + + +@pytest.mark.integration +def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_with_score_set): + """Default order: direct before related; within a group strongest evidence first, pathogenic before + benign at equal magnitude; the unclassified related measurement sorts last.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + strong_path = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_path], functional="abnormal", oddspaths_ratio=100.0) + + strong_benign = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, strong_benign, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[strong_benign], functional="normal", oddspaths_ratio=0.01) + + weak_path = _variant(session, score_set, 3, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, weak_path, assay_level="cdna"), nt, is_authoritative=True) + _calibration(session, score_set, variants=[weak_path], functional="abnormal", oddspaths_ratio=2.0) + + related = _variant(session, score_set, 4, data={"score_data": {"score": 1.0}}) + related_record = _record(session, related, assay_level="protein") + _link(session, related_record, prot, is_authoritative=True) + _link(session, related_record, nt) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert [m.variant_urn for m in result] == [strong_path.urn, strong_benign.urn, weak_path.urn, related.urn] + + +@pytest.mark.integration +def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): + assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py new file mode 100644 index 000000000..7d75739db --- /dev/null +++ b/tests/routers/test_clingen_allele.py @@ -0,0 +1,122 @@ +# ruff: noqa: E402 +"""Router tests for the ClinGen-allele measurements endpoint +(``GET /clingen-alleles/{caid}/measurements``, design §7.5). + +Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` +content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded +measurements are opt-in, and a private score set's measurement never leaks. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from sqlalchemy import select + +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from mavedb.view_models.allele_measurement import AlleleMeasurement +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + + +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): + """Give a variant a live coding-measured mapping record whose authoritative allele carries the + ClinGen id — the minimal shape for a direct measurement on that CA's page.""" + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) + record = MappingRecord( + variant_id=variant.id, + assay_level="cdna", + hgvs_assay_level="NM_000546.6:c.1216G>A", + mapping_api_version="test.0.0", + ) + session.add(record) + session.commit() + + measured = Allele(vrs_digest="cdna-digest", level="cdna", clingen_allele_id=clingen_allele_id) + session.add(measured) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) + session.commit() + + +def test_measurements_serialize(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + urn = f"{score_set['urn']}#1" + _seed_cdna_measurement(session, urn) + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + body = response.json() + assert len(body) == 1 + AlleleMeasurement.model_validate(body[0]) + assert body[0]["variantUrn"] == urn + assert body[0]["relationship"] == "direct" + assert body[0]["assayLevel"] == "cdna" + assert body[0]["assayLevelHgvs"] == "NM_000546.6:c.1216G>A" + assert body[0]["submittedHgvs"] == "c.1A>T" + assert body[0]["scoreSetUrn"] == score_set["urn"] + assert body[0]["isCurrent"] is True + assert "supersededByScoreSet" not in body[0] # dropped by exclude_none when current + assert response.headers["X-As-Of"] == "current" + + +def test_unknown_clingen_id_returns_empty_list(client, setup_router_db): + response = client.get("/api/v1/clingen-alleles/CA000/measurements") + assert response.status_code == 200 + assert response.json() == [] + + +def test_superseded_measurement_is_opt_in(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1") + + default = client.get("/api/v1/clingen-alleles/CA123/measurements") + assert default.status_code == 200 + assert default.json() == [] + + opted = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + assert opted.status_code == 200 + body = opted.json() + assert len(body) == 1 + assert body[0]["isCurrent"] is False + assert body[0]["supersededByScoreSet"] == newer["urn"] + + +def test_anonymous_cannot_see_private_measurement( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed_cdna_measurement(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + # A content-addressed ClinGen id is public, but a private score set's measurement is filtered out. + assert response.status_code == 200 + assert response.json() == [] From 41ada42c8a260d14f32af28365d510cfb439361e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 7 Jul 2026 22:58:42 -0700 Subject: [PATCH 65/93] feat(variants): add per-allele identity sidecar to variant detail envelope Replace the UI-unused memberRelations map on GET /variants/{urn} with an alleles sidecar keyed by VRS digest: one entry per linked allele (the record-scoped, all-levels set, sharing keys with annotations) carrying level, reference-frame HGVS, ClinGen id, and its relation to the measured allele. This lets the selection panel label annotations at every level by level + HGVS rather than by digest, without per-digest /alleles/{digest} fetches. --- src/mavedb/lib/variant_detail.py | 57 ++++++++++++++++++++---- src/mavedb/view_models/variant_detail.py | 35 +++++++++++---- tests/lib/test_variant_detail.py | 37 ++++++++++++--- tests/routers/test_variant.py | 22 +++++++-- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 0331c7d7a..30f3fbc3b 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -3,7 +3,8 @@ Assembles the two-tier envelope for a single variant: flat, UI-ergonomic assay fields (the assay-level HGVS pair, digest, ClinGen id) plus the spec-pure GA4GH ``CategoricalVariant`` (built on the fly by :mod:`lib.cat_vrs`) and, riding alongside it keyed by VRS digest, the MaveDB layer — -per-member relations and the digest-keyed external-annotation map (:mod:`lib.annotations`). Also +the per-allele identity sidecar (level / HGVS / ClinGen id / member→defining relation) and the +digest-keyed external-annotation map (:mod:`lib.allele_annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather than reading as current (design §9.2). @@ -52,6 +53,24 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one of the variant's linked alleles. + + Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and + ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are + what the UI labels the per-level annotation panel by — *never* the digest. ``relation`` is this + allele's relation to the measured (defining) allele; ``None`` when it *is* the measured allele, or + when the allele is not a Cat-VRS member. (The Cat-VRS relation is a *member*→defining code, hence + absent for the defining allele — see ``cat_vrs.CategoricalVariantTransit.member_relations``.) + """ + + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + relation: Optional[str] + + @dataclass(frozen=True) class VariantDetail: """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" @@ -68,12 +87,14 @@ class VariantDetail: assay_level_digest: Optional[str] clingen_allele_id: Optional[str] - # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside. + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside: the + # per-allele identity sidecar, keyed by VRS digest — one entry per linked allele (the record-scoped, + # all-levels set, sharing keys with `annotations`), carrying level + HGVS + ClinGen id + relation. molecular_representation: Optional[dict[str, Any]] mode: Optional[str] # CatVrsMode value — projection | reverse_translation - member_relations: dict[str, str] # member vrs_digest -> relation (member -> defining) + alleles: dict[str, AlleleIdentity] - # External annotations, keyed by VRS digest, joined to the Cat-VRS members. + # External annotations, keyed by VRS digest, joined to the Cat-VRS members / the alleles sidecar. annotations: dict[str, AlleleAnnotations] # Version standing — self-descriptive for a superseded variant. @@ -161,16 +182,36 @@ def get_variant_detail( annotations = get_allele_annotations(db, [link.allele for link in links], as_of=as_of) - # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). + # Spec-pure Cat-VRS built on the fly, plus the MaveDB layer (mode + per-member relations). The + # relations key the sidecar below; the defining allele is deliberately absent from them, so it + # gets relation=None. transit = categorical_variant_for_variant(db, variant.id, name=variant.urn or "", as_of=as_of) if transit is not None: molecular_representation = transit.categorical_variant.model_dump(mode="json", exclude_none=True) mode: Optional[str] = transit.mode.value - member_relations = {digest: relation.value for digest, relation in transit.member_relations.items()} + relations = transit.member_relations else: molecular_representation = None mode = None - member_relations = {} + relations = {} + + # The per-allele identity sidecar: one entry per linked allele, keyed by VRS digest (the same + # record-scoped, all-levels link set that seeds `annotations`, so the two maps share keys). Carries + # the molecular-identity facts the UI labels annotations by — level, reference-frame HGVS (exactly one + # of hgvs_g/c/p is populated per allele), ClinGen id — and the member→defining relation. + alleles: dict[str, AlleleIdentity] = {} + for link in links: + allele = link.allele + if allele.vrs_digest is None: + continue + + relation = relations.get(allele.vrs_digest) + alleles[allele.vrs_digest] = AlleleIdentity( + level=allele.level, + hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, + clingen_allele_id=allele.clingen_allele_id, + relation=relation.value if relation is not None else None, + ) return VariantDetail( urn=variant.urn or "", @@ -184,7 +225,7 @@ def get_variant_detail( clingen_allele_id=clingen_allele_id, molecular_representation=molecular_representation, mode=mode, - member_relations=member_relations, + alleles=alleles, annotations=annotations, is_current=superseding_score_set is None, superseded_by_score_set=superseding_score_set.urn if superseding_score_set is not None else None, diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 2c242f756..b4c5fa5b2 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -29,18 +29,37 @@ class Config: from_attributes = True +class AlleleIdentity(BaseModel): + """The MaveDB molecular-identity facts for one of the variant's linked alleles. + + An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame + HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). + ``relation`` is this allele's relation to the measured (defining) allele; ``null`` when it *is* the + measured allele, or when the allele is not a Cat-VRS member. + """ + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + relation: Optional[str] = None + + class Config: + from_attributes = True + + class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` - (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside keyed by VRS - digest: ``memberRelations`` (member→defining relation) and the ``annotations`` map. ``isCurrent`` - /``supersededByScoreSet`` let a superseded variant self-describe: ``supersededByScoreSet`` is the - superseding *score set*'s URN, not a variant URN. Supersession is versioned at the score-set level, - and a newer version may add, drop, or renumber variants — so there is no stable - superseding-*variant* pointer to hand back; a consumer resolves the current measurement by looking - this variant up within that score set. Absent fields are omitted. + (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside, keyed by VRS + digest: the ``alleles`` identity sidecar (per-allele ``level`` / ``hgvs`` / ``clingenAlleleId`` / + ``relation`` — one entry per linked allele, sharing keys with ``annotations``) and the + ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a superseded variant self-describe: + ``supersededByScoreSet`` is the superseding *score set*'s URN, not a variant URN. Supersession is + versioned at the score-set level, and a newer version may add, drop, or renumber variants — so there + is no stable superseding-*variant* pointer to hand back; a consumer resolves the current measurement + by looking this variant up within that score set. Absent fields are omitted. """ urn: str @@ -56,7 +75,7 @@ class VariantDetail(BaseModel): molecular_representation: Optional[dict[str, Any]] = None mode: Optional[str] = None - member_relations: dict[str, str] = {} + alleles: dict[str, AlleleIdentity] = {} annotations: dict[str, AlleleAnnotations] = {} diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 6cea44858..fcf98c31b 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -66,8 +66,16 @@ def _variant(session, score_set, suffix, *, data=None, hgvs_nt=None, hgvs_pro=No return variant -def _allele(session, digest, *, level="cdna", clingen_allele_id=None): - allele = Allele(vrs_digest=digest, level=level, post_mapped=_post_mapped(), clingen_allele_id=clingen_allele_id) +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_g=None, hgvs_c=None, hgvs_p=None): + allele = Allele( + vrs_digest=digest, + level=level, + post_mapped=_post_mapped(), + clingen_allele_id=clingen_allele_id, + hgvs_g=hgvs_g, + hgvs_c=hgvs_c, + hgvs_p=hgvs_p, + ) session.add(allele) session.commit() return allele @@ -134,8 +142,8 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): session, score_set, 1, data={"score_data": {"score": -2.3}}, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr" ) record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") - measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") - protein = _allele(session, "prot-digest", level="protein") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") _link(session, record, measured, is_authoritative=True) _link(session, record, protein) _vep(session, measured, "missense_variant") @@ -152,8 +160,19 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.mode == "projection" assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" + # The alleles sidecar carries one identity per linked allele, keyed by digest: level + + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation. + assert set(detail.alleles) == {"cdna-digest", "prot-digest"} + measured_identity = detail.alleles["cdna-digest"] + assert measured_identity.level == "cdna" + assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c + assert measured_identity.clingen_allele_id == "CA123" + assert measured_identity.relation is None # the defining allele has no relation to itself + protein_identity = detail.alleles["prot-digest"] + assert protein_identity.level == "protein" + assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p # The protein member is a translation of the measured coding allele (member -> defining). - assert detail.member_relations == {"prot-digest": "translation_of"} + assert protein_identity.relation == "translation_of" # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. assert set(detail.annotations) == {"cdna-digest", "prot-digest"} assert detail.annotations["cdna-digest"].vep is not None @@ -181,7 +200,11 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - assert detail.member_relations == {"cdna-digest": "encodes"} + # Defining protein allele has no relation to itself; the coding member encodes it. + assert detail.alleles["prot-digest"].level == "protein" + assert detail.alleles["prot-digest"].relation is None + assert detail.alleles["cdna-digest"].level == "cdna" + assert detail.alleles["cdna-digest"].relation == "encodes" @pytest.mark.integration @@ -201,7 +224,7 @@ def test_unmapped_variant_has_null_molecular_layer(session, setup_lib_db_with_sc assert detail.clingen_allele_id is None assert detail.molecular_representation is None assert detail.mode is None - assert detail.member_relations == {} + assert detail.alleles == {} assert detail.annotations == {} assert detail.is_current is True diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 365fe9ca7..8bb1c3503 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -64,8 +64,16 @@ def _seed_mapping(session, variant_urn): session.add(record) session.commit() - measured = Allele(vrs_digest="cdna-digest", level="cdna", post_mapped=_post_mapped(), clingen_allele_id="CA123") - protein = Allele(vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped()) + measured = Allele( + vrs_digest="cdna-digest", + level="cdna", + post_mapped=_post_mapped(), + clingen_allele_id="CA123", + hgvs_c="NM_000546.6:c.1216G>A", + ) + protein = Allele( + vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" + ) session.add_all([measured, protein]) session.commit() @@ -106,7 +114,15 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["mode"] == "projection" # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). assert body["molecularRepresentation"]["type"] == "CategoricalVariant" - assert body["memberRelations"] == {"prot-digest": "translation_of"} + # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a + # translation of the defining coding allele (which itself has no relation to itself). + assert body["alleles"]["cdna-digest"]["level"] == "cdna" + assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" + assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + assert body["alleles"]["prot-digest"]["level"] == "protein" + assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" + assert body["alleles"]["prot-digest"]["relation"] == "translation_of" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current From 031eaab3cf2fc29e7b1ef1054d24d2d24ae93f77 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 8 Jul 2026 12:34:24 -0700 Subject: [PATCH 66/93] =?UTF-8?q?perf(score-set-variants):=20split=20prote?= =?UTF-8?q?in=20projection=20into=20separate=20query=20to=20avoid=20O(N?= =?UTF-8?q?=C2=B2)=20planner=20trap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protein HGVS subquery uses DISTINCT ON, whose output cardinality is opaque to the planner (~18 estimated vs ~11.5k actual). Joined in, the planner buried it on the inner side of a nested loop and rescanned it in full once per variant — accounting for ~97% of the endpoint's total time. MATERIALIZED in a CTE fixed construction but not the per-variant rescan. Pulling the protein projection out as a standalone query and stitching it back by mapping_record_id in Python keeps both queries O(N) — the same batched-join pattern SQLAlchemy's selectin loader uses internally for the same class of cardinality estimation problem. --- src/mavedb/lib/score_set_variants.py | 93 ++++++++++++++-------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index 134a4eae2..c6bd8e382 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -1,8 +1,10 @@ """The lean whole-set view backing the score-set page. The score-set table, heatmap, and score/effect histograms all compose from one pre-chewed per-variant -dataset. This module assembles that dataset for an entire score set in a single bulk query and returns -one record per variant. +dataset. This module assembles that dataset for an entire score set in two O(N) bulk queries — a base +per-variant projection plus a standalone protein-HGVS projection stitched back in Python — and returns +one record per variant. (The protein projection is deliberately *not* a join: folded in, its opaque +cardinality drives the SQL planner into an O(N^2) per-variant rescan; see ``get_lean_score_set_variants``.) Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level @@ -92,50 +94,18 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # Per record, the forward-translated protein HGVS (hgvs_p) of its protein-level allele. - prot_link = aliased(MappingRecordAllele) - prot_allele = aliased(Allele) - prot_record = aliased(MappingRecord) - prot_variant = aliased(Variant) - protein_allele = ( - select( - prot_link.mapping_record_id.label("mapping_record_id"), - prot_allele.hgvs_p.label("protein_hgvs"), - ) - # DISTINCT ON -> at most one protein level row per record. - .distinct(prot_link.mapping_record_id) - # link -> allele: level and hgvs_p live on the *allele*. - .join(prot_allele, prot_allele.id == prot_link.allele_id) - # link -> record -> variant: the bridge to scoreset_id, to scope the subquery (next). - .join(prot_record, prot_record.id == prot_link.mapping_record_id) - .join(prot_variant, prot_variant.id == prot_record.variant_id) - # Scope to this score set so the subquery's cost scales with the response. - .where(prot_variant.score_set_id == score_set.id) - # Live links only. A live link implies a live parent record (ValidTime invariant), so this also - # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. - .where(prot_link.live_at(as_of)) - # Only the protein-level allele. - .where(prot_allele.level == AnnotationLayer.protein.value) - # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives - # per record — inert today (only one protein allele exists). - .order_by(prot_link.mapping_record_id, prot_allele.id) - # MATERIALIZED forces this to run exactly once. The planner badly under-counted the DISTINCT ON - # cardinality (est. ~5 vs ~2.8k actual) and, left as a plain subquery, buries it on the inner - # side of a nested loop, re-running the whole projection once per variant. That is O(N^2) in the - # score set size. Materializing collapses it back to O(N). - .cte("protein_allele") - .prefix_with("MATERIALIZED") - ) - - statement = ( + # The base per-variant projection: the four annotation joins that stay strictly 1:1 with the variant, + # so this whole query is a stream of O(N) index-scan lookups. MappingRecord.id rides along as the key + # the protein projection (a *separate* query, below) is stitched back on. + base_statement = ( select( Variant.urn, Variant.data, Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, + MappingRecord.id.label("mapping_record_id"), MappingRecord.hgvs_assay_level, - protein_allele.c.protein_hgvs.label("protein_hgvs"), Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, @@ -162,16 +132,49 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) - # The protein projection (materialized above), matched back by record id (a globally-unique PK -> - # the right protein row, no cross-set risk). LEFT: some rows have no protein projection - # (UTR/intronic) -> null. - .outerjoin(protein_allele, protein_allele.c.mapping_record_id == MappingRecord.id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) + # The protein projection runs as its own query, NOT a join into the base statement. Joined in, it forces + # an O(N^2) plan: the DISTINCT ON cardinality is opaque to the planner (est. ~18 vs ~11.5k actual), so it + # buried the projection on the inner side of a nested loop and rescanned it in full once per variant — + # ~97% of the endpoint's total time. MATERIALIZED in the query above fixes the *construction* but not the + # per-variant *rescan*. Pulling out and stitching these queries by mapping_record_id in Python enables both + # queries to run in O(N) time, making this function (and the endpoint it serves) considerably faster. + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_statement = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the query. + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the cost scales with the response. + .where(prot_variant.score_set_id == score_set.id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. + .where(prot_allele.level == AnnotationLayer.protein.value) + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + ) + # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records + # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. + protein_hgvs_by_record = {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + return [ LeanVariantRecord( variant_urn=row.urn, @@ -183,7 +186,7 @@ def get_lean_score_set_variants( hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), - protein_level_hgvs=_hgvs_field_for_str(row.protein_hgvs), + protein_level_hgvs=_hgvs_field_for_str(protein_hgvs_by_record.get(row.mapping_record_id)), ) - for row in db.execute(statement) + for row in db.execute(base_statement) ] From aafe524f2aa8b70057e62da4a85c1cb64473b0d2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 8 Jul 2026 22:08:55 -0700 Subject: [PATCH 67/93] feat(annotation): pair c/g projections via projection_group Add a nullable projection_group column to mapping_record_alleles and thread the coding/genomic pairing it records through the reverse- translation worker and the serving layer. - migration e7b2c9a1f4d3: add nullable projection_group (no index, no backfill; existing rows stay NULL and re-group on next re-map) - RT worker: consume the library's ProjectionPair list, stamp both links of a pair with a shared per-record group id, leave the protein apex ungrouped, and fold a pair member that equals the record's authoritative allele onto the existing link instead of duplicating it - lean variant view: replace assayLevelHgvs/proteinLevelHgvs with an assayLevel pointer plus a mapped MappedTriple (genomic/cdna/protein), filling the other nucleotide slot from the authoritative link's projection_group sibling via one indexed 1:1 join - variant detail: add derivation (authoritative/projection/candidate) and projection_of (the sibling's VRS digest) to AlleleIdentity Unpopulated projection_group (pre-RT data, protein apex) degrades gracefully to a null sibling / empty slot. --- ...jection_group_to_mapping_record_alleles.py | 37 ++ src/mavedb/lib/score_set_variants.py | 150 +++++-- src/mavedb/lib/variant_detail.py | 83 +++- src/mavedb/models/mapping_record_allele.py | 16 +- src/mavedb/view_models/lean_variant.py | 36 +- src/mavedb/view_models/variant_detail.py | 13 +- .../variant_processing/reverse_translation.py | 84 ++-- tests/helpers/util/annotation.py | 6 +- tests/lib/test_score_set_variants.py | 121 +++-- tests/lib/test_variant_detail.py | 97 +++- tests/routers/test_score_set.py | 30 +- tests/routers/test_variant.py | 25 +- .../test_reverse_translation.py | 417 ++++++++++++++---- 13 files changed, 916 insertions(+), 199 deletions(-) create mode 100644 alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py diff --git a/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py new file mode 100644 index 000000000..6d0d0ff7a --- /dev/null +++ b/alembic/versions/e7b2c9a1f4d3_add_projection_group_to_mapping_record_alleles.py @@ -0,0 +1,37 @@ +"""add projection_group to mapping_record_alleles + +Revision ID: e7b2c9a1f4d3 +Revises: f1a3c7e9b2d5 +Create Date: 2026-07-08 + +Adds a nullable ``projection_group`` integer column to ``mapping_record_alleles`` — a within-record +grouping key that pairs the coding and genomic links of one reverse-translation projection pair. +The two members of a pair share a value; the shared protein apex is NULL, as is every row written +before reverse translation runs. See the model and the reverse_translation worker for the full +semantics (per-record index, authoritative fold-in). + +No index is added: the per-URN serving fetch is already covered by +``ix_mapping_record_alleles_mapping_record_id`` and a record's link set is small. No backfill: +existing rows stay NULL and re-group lazily on the next re-map / reverse-translation run. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e7b2c9a1f4d3" +down_revision = "f1a3c7e9b2d5" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "mapping_record_alleles", + sa.Column("projection_group", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("mapping_record_alleles", "projection_group") diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index c6bd8e382..b2a2e060a 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -6,18 +6,25 @@ one record per variant. (The protein projection is deliberately *not* a join: folded in, its opaque cardinality drives the SQL planner into an O(N^2) per-variant rescan; see ``get_lean_score_set_variants``.) -Each record carries the HGVS in both frames the heatmap toggles between — the depositor-**submitted** -strings (``hgvs_nt``/``hgvs_pro``/``hgvs_splice``, the target frame) and the **mapped** assay-level -string (reference frame) — plus, riding alongside each string, the parsed ``position``/``ref``/``alt`` -block when the expression is a placeable simple substitution (the heatmap grid; ``None`` for -splice/indels/multivariants, which keep the string alone). The string is the canonical, lossless field; -the block is a convenience the client would otherwise derive by parsing. - -Because every mapped string now comes from the ``MappingRecord`` and the only allele we need is the -**authoritative** (measured) one — for its digest, ClinGen id, and VEP consequence — the query joins -the authoritative link only and so returns **one row per variant** (no allele fan-out to regroup). It -does not assemble Cat-VRS. ``as_of`` reconstructs the annotation layer at a past instant over -the variant's immutable scores/submitted HGVS; it defaults to the currently-live rows. +Each record carries the **submitted** HGVS the heatmap's frame toggle needs (``hgvs_nt``/``hgvs_pro``/ +``hgvs_splice``, the depositor's target frame) plus the **mapped** (reference frame) representation as a +``MappedTriple`` — one canonical HGVS per level (``genomic`` / ``cdna`` / ``protein``) and an +``assay_level`` pointer naming which slot is the measured one. ``mapped[assay_level]`` is the measured +representation. Only the *canonical* projection is served here — never the reverse-translation +fan-out (that lives in the detail endpoint), so a nucleotide assay fills all three slots while a protein +assay fills only ``protein`` (``cdna``/``genomic`` stay ``None`` — the c/g fan-out is ambiguous, so no +pick is fabricated). Each HGVS rides as an ``HgvsField``: the string (canonical, lossless) plus a parsed +``position``/``ref``/``alt`` block when it is a placeable simple substitution (the heatmap grid; ``None`` +for splice/indels/multivariants). + +The canonical nucleotide pair comes from the **authoritative** (measured) allele — its digest, ClinGen +id, and VEP consequence — plus one indexed join to its ``projection_group`` sibling at the other +nucleotide level (still **one row per variant**: a group has ≤2 members, so the authoritative has ≤1 +sibling). The protein slot is the separate ``DISTINCT ON`` subquery. Where ``projection_group`` is +unpopulated (pre-reverse-translation data) the sibling join returns null → the other nucleotide slot +stays ``None`` and improves as data is re-processed. This view does not assemble Cat-VRS. ``as_of`` +reconstructs the annotation layer at a past instant over the variant's immutable scores/submitted HGVS; +it defaults to the currently-live rows. """ from dataclasses import dataclass @@ -53,15 +60,34 @@ class HgvsField: alt: Optional[str] = None -# TODO#784: Add a canonical nucleotide level projection in parallel with the canonical protein level projection. +@dataclass(frozen=True) +class MappedTriple: + """The mapped (reference-frame) HGVS at each level — the canonical projection of the measured change. + + One slot per level, each an :class:`HgvsField` or ``None``. A **nucleotide** assay populates all + three (``mapped[assay_level]`` is the measured slot, the other nucleotide slot is the authoritative + allele's ``projection_group`` sibling, ``protein`` is the apex). A **protein** assay populates only + ``protein`` — the c/g fan-out is ambiguous, so ``cdna``/``genomic`` stay ``None`` rather than + fabricate a canonical pick. A slot is also ``None`` when its source is simply absent (an unmapped + variant; a projection that failed; pre-reverse-translation data with no sibling recorded). + + ``cdna`` is the level-invariant search key — populated even when ``assay_level`` is ``genomic``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + @dataclass(frozen=True) class LeanVariantRecord: """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions - (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the two mapped - representations: ``assay_level_hgvs`` (the measured level) and ``protein_level_hgvs`` (the protein level). - For a protein assay the two mapped fields coincide. Any field is ``None`` when its source is absent - (unmapped variant → null mapped fields; a level the variant does not carry → null slot).""" + (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the mapped + (reference-frame) representation as an ``assay_level`` pointer (an ``AnnotationLayer`` value naming + the measured/canonical slot) plus a ``mapped`` :class:`MappedTriple`. ``mapped[assay_level]`` is the + measured representation; ``mapped.cdna`` is the level-invariant search key. Any field is ``None`` + when its source is absent (unmapped variant → null ``assay_level`` + empty triple).""" variant_urn: str score: Optional[float] @@ -71,8 +97,8 @@ class LeanVariantRecord: hgvs_nt: Optional[HgvsField] hgvs_pro: Optional[HgvsField] hgvs_splice: Optional[HgvsField] - assay_level_hgvs: Optional[HgvsField] - protein_level_hgvs: Optional[HgvsField] + assay_level: Optional[str] + mapped: MappedTriple def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: @@ -86,6 +112,43 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) +def _mapped_triple( + *, + assay_level: Optional[str], + assay_level_hgvs: Optional[str], + sibling_level: Optional[str], + sibling_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> MappedTriple: + """Assemble the canonical :class:`MappedTriple` for one variant from its per-row query fields. + + - Measured slot (``mapped[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). + - Other nucleotide slot ← the authoritative link's ``projection_group`` sibling (``sibling_level`` / + ``sibling_hgvs``), populated only on a nucleotide assay and only where the pairing is recorded. + - ``protein`` slot ← the separate protein-apex subquery. + + A **protein** assay fills only ``protein`` (the measured slot is protein itself); its nucleotide + fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately left ``None``. An unmapped variant + (``assay_level is None``) yields an empty triple. + """ + slots: dict[str, Optional[HgvsField]] = {} + # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. + if assay_level == AnnotationLayer.protein.value: + slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + elif assay_level in (AnnotationLayer.cdna.value, AnnotationLayer.genomic.value): + slots[assay_level] = _hgvs_field_for_str(assay_level_hgvs) + # The other nucleotide level, from the projection_group sibling (null where unpopulated). + if sibling_level is not None: + slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) + slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(protein_hgvs) + + return MappedTriple( + genomic=slots.get(AnnotationLayer.genomic.value), + cdna=slots.get(AnnotationLayer.cdna.value), + protein=slots.get(AnnotationLayer.protein.value), + ) + + def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: @@ -94,9 +157,15 @@ def get_lean_score_set_variants( histogram still see them. ``as_of`` reconstructs the annotation layer at a past instant (submitted HGVS and scores are immutable and unaffected); it defaults to the currently-live rows. """ - # The base per-variant projection: the four annotation joins that stay strictly 1:1 with the variant, - # so this whole query is a stream of O(N) index-scan lookups. MappingRecord.id rides along as the key - # the protein projection (a *separate* query, below) is stitched back on. + # The other-nucleotide sibling of the authoritative link: same mapping record + same + # projection_group, but NOT authoritative (so it is the *other* member of the ≤2-member group). Its + # allele carries the canonical HGVS at the level the assay was not measured at. See the join below. + sibling_link = aliased(MappingRecordAllele) + sibling_allele = aliased(Allele) + + # The base per-variant projection: 1:1 annotation joins, so this whole query is a stream of O(N) + # index-scan lookups. MappingRecord.id rides along as the key the protein projection (a *separate* + # query, below) is stitched back on. base_statement = ( select( Variant.urn, @@ -105,10 +174,15 @@ def get_lean_score_set_variants( Variant.hgvs_pro, Variant.hgvs_splice, MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, MappingRecord.hgvs_assay_level, Allele.vrs_digest, Allele.clingen_allele_id, VepAlleleConsequence.functional_consequence, + sibling_allele.level.label("sibling_level"), + # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + # coalesce yields its canonical string regardless of which nucleotide level it sits at. + func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), ) # Variant -> its live mapping record. LEFT join so unmapped variants stay (with null mapped fields) # for the table + score histogram. live_at rides in the ON clause, never a WHERE: a WHERE would @@ -132,6 +206,24 @@ def get_lean_score_set_variants( .outerjoin( VepAlleleConsequence, and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)) ) + # The authoritative link's projection_group sibling — the canonical projection at the *other* + # nucleotide level. UNLIKE the protein subquery, this stays a direct 1:1 outer join: a group has + # ≤2 members and the authoritative link is one of them, so at most one non-authoritative sibling + # shares its (mapping_record_id, projection_group). The mapping_record_id predicate rides the + # existing ix_mapping_record_alleles_mapping_record_id index; within a record only a handful of + # links are scanned, so no O(N^2) rescan (contrast the opaque-cardinality protein subquery below). + # A NULL authoritative group (protein assay, or pre-reverse-translation data) matches nothing — + # SQL equality on NULL is never true — so the sibling stays null and the slot degrades gracefully. + .outerjoin( + sibling_link, + and_( + sibling_link.mapping_record_id == MappingRecord.id, + sibling_link.projection_group == MappingRecordAllele.projection_group, + sibling_link.is_authoritative.is_(False), + sibling_link.live_at(as_of), + ), + ) + .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) # The anchor: just this score set's variants. .where(Variant.score_set_id == score_set.id) # Natural table order: variant number (the integer after '#'), id breaks ties stably. @@ -173,7 +265,9 @@ def get_lean_score_set_variants( ) # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. - protein_hgvs_by_record = {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + protein_hgvs_by_record: dict[int, str] = { + row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement) + } return [ LeanVariantRecord( @@ -185,8 +279,14 @@ def get_lean_score_set_variants( hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), - assay_level_hgvs=_hgvs_field_for_str(row.hgvs_assay_level), - protein_level_hgvs=_hgvs_field_for_str(protein_hgvs_by_record.get(row.mapping_record_id)), + assay_level=row.assay_level, + mapped=_mapped_triple( + assay_level=row.assay_level, + assay_level_hgvs=row.hgvs_assay_level, + sibling_level=row.sibling_level, + sibling_hgvs=row.sibling_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ), ) for row in db.execute(base_statement) ] diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 30f3fbc3b..e833545a8 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from datetime import datetime +from enum import Enum from typing import Any, Optional from sqlalchemy import select @@ -53,22 +54,54 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at — its confidence/provenance axis. + + Distinct from (orthogonal to) the Cat-VRS ``relation`` axis: ``relation`` is *structural* (which + level relates to which, member→defining), whereas ``derivation`` is *epistemic* (how much to trust + the representation). See the design's "Semantics note — two axes, keep them separate". Derived at + serialization time from ``is_authoritative`` + the assay level — no stored column. + """ + + # The assay's actual measurement (the authoritative link). Precise by definition. + AUTHORITATIVE = "authoritative" + # Deterministic and precise. Every allele derived from a *nucleotide* measurement is a projection: + # nucleotide↔nucleotide and nucleotide→protein are both deterministic given (assembly, transcript). + PROJECTION = "projection" + # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous + # codons). One member of the fanned-out equivalence class, not a precise coordinate. + CANDIDATE = "candidate" + + @dataclass(frozen=True) class AlleleIdentity: """The MaveDB molecular-identity facts for one of the variant's linked alleles. Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are - what the UI labels the per-level annotation panel by — *never* the digest. ``relation`` is this - allele's relation to the measured (defining) allele; ``None`` when it *is* the measured allele, or - when the allele is not a Cat-VRS member. (The Cat-VRS relation is a *member*→defining code, hence - absent for the defining allele — see ``cat_vrs.CategoricalVariantTransit.member_relations``.) + what the UI labels the per-level annotation panel by — *never* the digest. + + Three axes ride here, deliberately independent: + + - ``relation`` (Cat-VRS, structural): this allele's member→defining relation + (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured + allele, or when the allele is not a Cat-VRS member. Sourced from + ``cat_vrs.CategoricalVariantTransit.member_relations``. + - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / + candidate. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide + assay has ``relation=translation_of`` but ``derivation=projection``, whereas the same-shaped + member of a protein assay is ``derivation=candidate``). + - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* + member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), + for pre-reverse-translation data, and where a level's projection failed. """ level: Optional[str] hgvs: Optional[str] clingen_allele_id: Optional[str] relation: Optional[str] + derivation: Optional[str] + projection_of: Optional[str] @dataclass(frozen=True) @@ -137,6 +170,29 @@ def _classifications_for_variant( ] +def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> AlleleDerivation: + """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. + + The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* + ambiguous boundary in the stack — protein → codon: + + - **nucleotide assay** (``assay_level`` genomic/cdna) → ``projection`` for all derived alleles. + nucleotide↔nucleotide and nucleotide→protein are deterministic, so the whole equivalence class + is precise. + - **protein assay** (``assay_level`` protein) → ``candidate`` for the derived (nucleotide) fan-out. + Reverse translation is genuinely ambiguous. + + Only ``assay_level`` (not the allele's own level) distinguishes the two, because on a nucleotide + assay every derived level is deterministic and on a protein assay every derived allele is part of + the ambiguous nucleotide fan-out. + """ + if is_authoritative: + return AlleleDerivation.AUTHORITATIVE + if assay_level == AnnotationLayer.protein.value: + return AlleleDerivation.CANDIDATE + return AlleleDerivation.PROJECTION + + def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is @@ -199,18 +255,37 @@ def get_variant_detail( # record-scoped, all-levels link set that seeds `annotations`, so the two maps share keys). Carries # the molecular-identity facts the UI labels annotations by — level, reference-frame HGVS (exactly one # of hgvs_g/c/p is populated per allele), ClinGen id — and the member→defining relation. + # Within-record projection grouping: projection_group -> the digests sharing it. A link's projection + # sibling is the ≤1 *other* digest in its group (the c↔g pair); the protein apex and any pre-RT link + # carry a NULL group and so appear in no bucket (projection_of stays None for them). + group_digests: dict[int, list[str]] = {} + for link in links: + if link.projection_group is not None and link.allele.vrs_digest is not None: + group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + alleles: dict[str, AlleleIdentity] = {} for link in links: allele = link.allele if allele.vrs_digest is None: continue + # The projection sibling: the other digest in this link's projection_group, if any. A group has + # ≤2 members, so there is at most one sibling — this allele's projection partner. + projection_of: Optional[str] = None + if link.projection_group is not None: + projection_of = next( + (digest for digest in group_digests.get(link.projection_group, []) if digest != allele.vrs_digest), + None, + ) + relation = relations.get(allele.vrs_digest) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, relation=relation.value if relation is not None else None, + derivation=_derivation_for(is_authoritative=link.is_authoritative, assay_level=assay_level).value, + projection_of=projection_of, ) return VariantDetail( diff --git a/src/mavedb/models/mapping_record_allele.py b/src/mavedb/models/mapping_record_allele.py index 7bfbd7885..5179877c2 100644 --- a/src/mavedb/models/mapping_record_allele.py +++ b/src/mavedb/models/mapping_record_allele.py @@ -31,7 +31,21 @@ class MappingRecordAllele(ValidTime, Base): # synonymous translator. The flag lives on the link rather than on # ``Allele`` because the same VRS allele can be authoritative for one # mapping record and translator-derived for another. - is_authoritative = Column(Boolean, nullable=False, default=False) + is_authoritative: Mapped[bool] = Column(Boolean, nullable=False, default=False) + # Within-record projection grouping. The reverse-translation job fans a variant's protein + # consequence out into projection pairs, each a coding/genomic pair (the same change expressed + # at two levels). The two links of one pair share a ``projection_group`` value + # (an integer index, 0..N-1, local to this mapping record), so we can reconstruct projections + # onto transcripts for more precise provenance in consumers. ``NULL`` for links that belong to no + # pair: the shared protein apex, and any link written before reverse translation has run. + # + # It is a grouping key, NOT a cross-record identity — regenerated whenever a record is + # superseded and re-linked. Identity lives in the allele VRS digests; two records that happen to + # share a transcript and codon re-encode the pairing independently. Serving resolves the + # canonical projection by pairing this with ``is_authoritative``: the authoritative (measured) + # allele's group gathers its sibling links, yielding its c/g projection. See the RT job for the + # group-assignment and authoritative fold-in logic. + projection_group = Column(Integer, nullable=True) mapping_record: Mapped["MappingRecord"] = relationship("MappingRecord", back_populates="allele_links") allele: Mapped["Allele"] = relationship("Allele", back_populates="mapping_record_links") diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 029a217a6..31f163524 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -3,8 +3,9 @@ The pydantic serialization boundary over the ``lib.score_set_variants`` transit dataclasses. ``from_attributes`` lets the route return the ``LeanVariantRecord`` dataclasses directly and have FastAPI's ``response_model`` coerce them (field names line up); aliases camelize per the shared base config, so clients see -``variantUrn`` / ``assayLevelHgvs`` etc. ``response_model_exclude_none`` drops absent fields, so a slot -with no parsed block serializes as just ``{"hgvs": "..."}``. +``variantUrn`` / ``assayLevel`` / ``mapped`` etc. ``response_model_exclude_none`` drops absent fields, so a +slot with no parsed block serializes as just ``{"hgvs": "..."}`` and a null mapped level drops out of the +``mapped`` object entirely. """ from typing import Optional @@ -28,15 +29,32 @@ class Config: from_attributes = True +class MappedTriple(BaseModel): + """The mapped (reference-frame) HGVS keyed by level — the canonical projection of the measured change. + + One slot per level. A nucleotide assay populates all three (``mapped[assayLevel]`` is the measured + slot; ``cdna`` is the level-invariant search key, present even when ``assayLevel`` is ``genomic``); a + protein assay populates only ``protein`` (the ambiguous c/g fan-out is not fabricated). Null slots are + omitted under ``response_model_exclude_none``. + """ + + genomic: Optional[HgvsField] = None + cdna: Optional[HgvsField] = None + protein: Optional[HgvsField] = None + + class Config: + from_attributes = True + + class LeanVariant(BaseModel): """One pre-chewed per-variant record feeding the score-set table, heatmap, and histograms. ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed - annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) and - the mapped ``assayLevelHgvs`` (reference frame) carry both sides of the heatmap's frame toggle, each - with an optional parsed block for the level toggle. ``proteinLevelHgvs`` is the mapped protein - representation (distinct from the *submitted* ``hgvsPro``); for a protein assay it coincides with - ``assayLevelHgvs``. Fields are omitted when null. + annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry + the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``AnnotationLayer`` value) naming + the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` + the level-invariant search key. Fields are omitted when null. """ variant_urn: str @@ -47,8 +65,8 @@ class LeanVariant(BaseModel): hgvs_nt: Optional[HgvsField] = None hgvs_pro: Optional[HgvsField] = None hgvs_splice: Optional[HgvsField] = None - assay_level_hgvs: Optional[HgvsField] = None - protein_level_hgvs: Optional[HgvsField] = None + assay_level: Optional[str] = None + mapped: MappedTriple = MappedTriple() class Config: from_attributes = True diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index b4c5fa5b2..1ae03def8 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -34,14 +34,23 @@ class AlleleIdentity(BaseModel): An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). - ``relation`` is this allele's relation to the measured (defining) allele; ``null`` when it *is* the - measured allele, or when the allele is not a Cat-VRS member. + + Three independent axes: + - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured + allele, or when the allele is not a Cat-VRS member. + - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, + precise) / ``candidate`` (reverse-translation, ambiguous). Orthogonal to ``relation`` — never + conflate them. + - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of + its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. """ level: Optional[str] = None hgvs: Optional[str] = None clingen_allele_id: Optional[str] = None relation: Optional[str] = None + derivation: Optional[str] = None + projection_of: Optional[str] = None class Config: from_attributes = True diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index d7f17e919..8d861270a 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -1,10 +1,19 @@ """Reverse translation worker job — builds cross-level HGVS equivalence classes. For each mapped variant in a score set, calls construct_equivalent_variants from -the variant-annotation library to produce all coding and genomic HGVS candidates -encoding the same protein consequence, plus that protein consequence itself. The -candidates (coding, genomic, and the protein) are written as non-authoritative -Allele rows linked to the existing MappingRecord via MappingRecordAllele. +the variant-annotation library to produce the coding/genomic HGVS candidates +encoding the same protein consequence, plus that protein consequence itself. Each +candidate is written as a non-authoritative Allele row linked to the existing +MappingRecord via MappingRecordAllele. + +The library returns the equivalence class as a list of ProjectionPair objects, +each one projection pair (a coding candidate and its deterministic genomic projection +— the same change at two levels). The job preserves that pairing by stamping both +links of a projection pair with a shared per-record ``projection_group`` id; the protein +apex is shared across all projection pairs and carries no group. Where a projection pair member +equals the record's authoritative (measured) allele, its group is folded onto the +existing authoritative link rather than duplicated, so the canonical c/g projection +is resolvable from the authoritative allele's group downstream. """ import asyncio @@ -355,19 +364,22 @@ async def reverse_translate_variants_for_score_set( # superseded atomically — retire and insert share a timestamp with no gap. new_links: list[MappingRecordAllele] = [] - # The live authoritative (record, allele) pairs. A derived candidate that equals a record's - # authoritative allele is not linked again. - authoritative_pairs: set[tuple[int, int]] = { - (record_id, allele_id) - for record_id, allele_id in job_manager.db.execute( - select(MappingRecordAllele.mapping_record_id, MappingRecordAllele.allele_id) + # The live authoritative links for the current records, keyed by (record_id, allele_id). + # Held as ORM objects (not a bare pair set) because the authoritative fold-in updates them in + # place: when an RT candidate's allele equals a record's authoritative (measured) allele, that + # link already exists (is_authoritative=True, projection_group NULL, written by the mapping job) + # and uq_mapping_record_alleles_live forbids a derived duplicate — so the candidate's group is + # stamped onto this existing link instead of inserting a new one. Modifying the loaded object + # emits an UPDATE on flush; supersede_live_where at the end only retires derived links, leaving + # the authoritative link (and its freshly stamped group) intact. + authoritative_links: dict[tuple[int, int], MappingRecordAllele] = { + (link.mapping_record_id, link.allele_id): link + for link in job_manager.db.scalars( + select(MappingRecordAllele) .where(MappingRecordAllele.is_authoritative.is_(True)) .where(MappingRecordAllele.current) .where(MappingRecordAllele.mapping_record_id.in_(current_record_ids)) - ) - .tuples() - .all() - if record_id is not None and allele_id is not None + ).all() } for result in results: @@ -378,17 +390,28 @@ async def reverse_translate_variants_for_score_set( # Dedup by vrs_digest per mapping record to avoid duplicate links. seen_digests: set[str] = set() failed_candidates: list[dict[str, str]] = [] - candidates: list[tuple[str, AnnotationLayer, str]] = [ - (hgvs_g, AnnotationLayer.genomic, "hgvs_g") for hgvs_g in result.hgvs_g_candidates - ] + [(hgvs_c, AnnotationLayer.cdna, "hgvs_c") for hgvs_c in result.hgvs_c_candidates] - # Emit the protein consequence as the protein-level member of the equivalence set. - # Prediction parens (p.(Ala222Val)) are stripped before translation and storage. - # None for protein-assay inputs where the protein is already the authoritative allele. + # Flatten the projection pairs into a work list of (hgvs, level, field, projection_group) members, + # preserving the coding↔genomic pairing the library emits. Each ProjectionPair's coding and + # genomic members share the pair's per-record group id — its index in the equivalence class + # (0..N-1). hgvs_c is always present (the pair key); hgvs_g is None when the c→g projection + # failed, yielding a well-formed one-member (coding only) group rather than a desync. The + # group id is assigned once per pair here, so the two members are guaranteed to carry the + # same id even though they translate and link independently below. + members: list[tuple[str, AnnotationLayer, str, int | None]] = [] + for group_id, pair in enumerate(result.projection_pairs): + members.append((pair.hgvs_c, AnnotationLayer.cdna, "hgvs_c", group_id)) + if pair.hgvs_g is not None: + members.append((pair.hgvs_g, AnnotationLayer.genomic, "hgvs_g", group_id)) + + # The protein consequence is the apex of the equivalence set — shared across every + # pair, a member of none — so it carries no group (None). Prediction parens + # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay + # inputs, where the protein is already the authoritative allele. if result.hgvs_p: - candidates.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p")) + members.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p", None)) - for hgvs, level, hgvs_field in candidates: + for hgvs, level, hgvs_field, projection_group in members: # A candidate may not be translatable (intronic projection, malformed expression). # Track per-candidate failures; a variant fails only if *all* candidates fail. try: @@ -415,14 +438,20 @@ async def reverse_translate_variants_for_score_set( allele = get_or_create_allele(job_manager.db, draft_allele) job_manager.db.flush() - if (rec.id, allele.id) in authoritative_pairs: - continue # already linked + authoritative_link = authoritative_links.get((rec.id, allele.id)) + if authoritative_link is not None: + # Authoritative fold-in: this member is the record's measured allele, already + # linked authoritatively. Stamp it's projection group to preserve the c↔g pairing, + # and skip creating a new derived link. + authoritative_link.projection_group = projection_group + continue new_links.append( MappingRecordAllele( mapping_record_id=rec.id, allele_id=allele.id, is_authoritative=False, + projection_group=projection_group, ) ) candidate_count += 1 @@ -430,8 +459,11 @@ async def reverse_translate_variants_for_score_set( annotation_counts["alleles_created"] += candidate_count annotation_metadata = { "hgvs_input": result.input.hgvs, - "hgvs_c_candidates": result.hgvs_c_candidates, - "hgvs_g_candidates": result.hgvs_g_candidates, + # Serializable projection of the projection pairs (the pairing the links now encode). + "candidates": [ + {"hgvs_c": pair.hgvs_c, "hgvs_g": pair.hgvs_g, "variant_type": pair.variant_type} + for pair in result.projection_pairs + ], "hgvs_p": result.hgvs_p, "alleles_created": candidate_count, "failed_candidates": failed_candidates, diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index 7e80a7c73..f845cdc1c 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -47,6 +47,7 @@ class AlleleSpec: hgvs_g: Optional[str] = None post_mapped: Optional[dict] = None vep_consequence: Optional[str] = None + projection_group: Optional[int] = None clinvar_control_ids: Sequence[int] = field(default_factory=tuple) gnomad_variant_ids: Sequence[int] = field(default_factory=tuple) @@ -99,7 +100,10 @@ def seed_mapping_record( links: list[Any] = [ MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=spec.is_authoritative + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=spec.is_authoritative, + projection_group=spec.projection_group, ) ] if spec.vep_consequence is not None: diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 22f4f43d0..4c0cbb5b0 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -15,7 +15,7 @@ pytest.importorskip("psycopg2") -from mavedb.lib.score_set_variants import HgvsField, get_lean_score_set_variants +from mavedb.lib.score_set_variants import HgvsField, MappedTriple, get_lean_score_set_variants from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele @@ -46,12 +46,14 @@ def _variant(session, score_set, suffix, *, score=None, hgvs_nt=None, hgvs_pro=N return variant -def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_p=None): +def _allele(session, digest, *, level="cdna", clingen_allele_id=None, hgvs_c=None, hgvs_g=None, hgvs_p=None): allele = Allele( vrs_digest=digest, level=level, post_mapped={"type": "Allele"}, clingen_allele_id=clingen_allele_id, + hgvs_c=hgvs_c, + hgvs_g=hgvs_g, hgvs_p=hgvs_p, ) session.add(allele) @@ -72,9 +74,13 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None, vali return record -def _link(session, record, allele, *, is_authoritative=True, valid_from=None): +def _link(session, record, allele, *, is_authoritative=True, projection_group=None, valid_from=None): link = MappingRecordAllele( - mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative, valid_from=valid_from + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + valid_from=valid_from, ) session.add(link) session.commit() @@ -91,15 +97,20 @@ def _consequence(session, allele, value): @pytest.mark.integration -def test_nucleotide_assay(session, setup_lib_db_with_score_set): - """Coding assay: submitted nt+pro from the variant, mapped coding string from the record, and the - digest/ClinGen id/consequence off the authoritative allele. Each parseable string gets its block.""" +def test_nucleotide_assay_measured_at_cdna(session, setup_lib_db_with_score_set): + """Coding assay: submitted nt+pro from the variant; the mapped triple carries the measured cdna slot + (from the record), its projection_group genomic sibling, and the protein apex; digest/ClinGen + id/consequence off the authoritative allele. Each parseable string gets its block.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T", hgvs_pro="p.Thr1Ser") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True) + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is a member of no pair (group None). + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, is_authoritative=False, projection_group=0) _link(session, record, protein, is_authoritative=False) _consequence(session, measured, "missense_variant") @@ -113,17 +124,45 @@ def test_nucleotide_assay(session, setup_lib_db_with_score_set): assert record_out.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") assert record_out.hgvs_pro == HgvsField(hgvs="p.Thr1Ser", position=1, ref="Thr", alt="Ser") assert record_out.hgvs_splice is None - assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") - # The mapped protein comes off the protein-level member allele's hgvs_p, not the measured coding allele. - assert record_out.protein_level_hgvs == HgvsField( - hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr" + # assayLevel names the measured slot; the triple fills all three levels. + assert record_out.assay_level == "cdna" + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), ) @pytest.mark.integration -def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db_with_score_set): - """Protein assay: the mapped assay-level string is a p. expression, parsed into a protein block; the - digest comes off the authoritative protein allele.""" +def test_nucleotide_assay_measured_at_genomic(session, setup_lib_db_with_score_set): + """Genomic assay: the measured slot is genomic (from the record); the cdna slot — the level-invariant + search key — comes off the projection_group sibling even though the assay was measured at genomic.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=0.1, hgvs_nt="g.7676154C>T") + record = _record(session, variant, assay_level="genomic", hgvs_assay_level="NC_000017.11:g.7676154C>T") + measured = _allele(session, "gen-digest", level="genomic", clingen_allele_id="CA9") + cdna = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, cdna, is_authoritative=False, projection_group=0) + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "genomic" + assert record_out.assay_level_digest == "gen-digest" + # Measured slot is genomic (from the record); cdna (search key) comes off the sibling allele. + assert record_out.mapped == MappedTriple( + genomic=HgvsField(hgvs="NC_000017.11:g.7676154C>T", position=7676154, ref="C", alt="T"), + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) + + +@pytest.mark.integration +def test_protein_assay_serves_only_the_protein_slot(session, setup_lib_db_with_score_set): + """Protein assay: the measured slot *is* protein, so only the protein slot is populated; the cdna and + genomic slots stay null (the c/g fan-out is ambiguous — no canonical pick is fabricated).""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, score=1.0, hgvs_pro="p.Ala406Thr") record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") @@ -131,15 +170,43 @@ def test_protein_assay_maps_assay_level_to_a_protein_block(session, setup_lib_db session, "prot-digest", level="protein", clingen_allele_id="PA9", hgvs_p="NP_000537.3:p.Ala406Thr" ) _link(session, record, measured, is_authoritative=True) + # A candidate fan-out member (cdna) with its own projection pair group — the protein authoritative link is + # in no group, so this candidate must NOT be pulled into the canonical triple. + candidate = _allele(session, "cand-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + _link(session, record, candidate, is_authoritative=False, projection_group=0) [record_out] = get_lean_score_set_variants(session, score_set) + assert record_out.assay_level == "protein" assert record_out.assay_level_digest == "prot-digest" assert record_out.clingen_allele_id == "PA9" - block = HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") - # For a protein assay the measured allele *is* the protein one, so the two mapped fields coincide. - assert record_out.assay_level_hgvs == block - assert record_out.protein_level_hgvs == block + # Only the protein slot; cdna/genomic stay null despite the candidate fan-out being linked. + assert record_out.mapped == MappedTriple( + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr") + ) + + +@pytest.mark.integration +def test_unpopulated_projection_group_degrades_to_a_null_sibling(session, setup_lib_db_with_score_set): + """Pre-reverse-translation data (no projection_group on the authoritative link): the sibling join + returns nothing, so the other nucleotide slot stays null — today's behavior — while the measured and + protein slots still populate. Still one row per variant.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, score=-2.3, hgvs_nt="c.1A>T") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # projection_group defaults to None + _link(session, record, protein, is_authoritative=False) + + [record_out] = get_lean_score_set_variants(session, score_set) + + assert record_out.assay_level == "cdna" + # Measured cdna + protein apex populate; genomic is null (no recorded sibling — graceful degradation). + assert record_out.mapped == MappedTriple( + cdna=HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A"), + protein=HgvsField(hgvs="NP_000537.3:p.Ala406Thr", position=406, ref="Ala", alt="Thr"), + ) @pytest.mark.integration @@ -157,7 +224,8 @@ def test_hgvs_string_rides_even_when_unparseable(session, setup_lib_db_with_scor assert record_out.hgvs_nt == HgvsField(hgvs="NM_000546.6:c.[197A>G;472T>C]") assert record_out.hgvs_nt.position is None assert record_out.hgvs_splice == HgvsField(hgvs="c.122-6T>A") - assert record_out.assay_level_hgvs == HgvsField(hgvs="c.76_78del") + # The measured cdna slot carries the (unparseable) indel string alone. + assert record_out.mapped.cdna == HgvsField(hgvs="c.76_78del") @pytest.mark.integration @@ -174,8 +242,8 @@ def test_unmapped_variant_has_null_mapped_fields(session, setup_lib_db_with_scor assert record_out.consequence is None assert record_out.clingen_allele_id is None assert record_out.assay_level_digest is None - assert record_out.assay_level_hgvs is None - assert record_out.protein_level_hgvs is None + assert record_out.assay_level is None + assert record_out.mapped == MappedTriple() # empty triple — no slots populated @pytest.mark.integration @@ -191,7 +259,7 @@ def test_no_vep_consequence_is_none(session, setup_lib_db_with_score_set): assert record_out.consequence is None assert record_out.assay_level_digest == "cdna-digest" - assert record_out.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + assert record_out.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") @pytest.mark.integration @@ -225,11 +293,11 @@ def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_d [current] = get_lean_score_set_variants(session, score_set) assert current.assay_level_digest == "new-digest" - assert current.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") + assert current.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.500A>T", position=500, ref="A", alt="T") [historical] = get_lean_score_set_variants(session, score_set, as_of=T1 - timedelta(days=1)) assert historical.assay_level_digest == "old-digest" - assert historical.assay_level_hgvs == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") + assert historical.mapped.cdna == HgvsField(hgvs="NM_000546.6:c.1216G>A", position=1216, ref="G", alt="A") # Submitted HGVS and score are immutable — unaffected by as_of. assert historical.hgvs_nt == HgvsField(hgvs="c.1A>T", position=1, ref="A", alt="T") assert historical.score == -2.3 @@ -237,4 +305,5 @@ def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_d # Before anything was mapped, the variant still appears with null mapped fields. [pre_mapping] = get_lean_score_set_variants(session, score_set, as_of=T0 - timedelta(days=1)) assert pre_mapping.assay_level_digest is None - assert pre_mapping.assay_level_hgvs is None + assert pre_mapping.assay_level is None + assert pre_mapping.mapped == MappedTriple() diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index fcf98c31b..fa193b53d 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -93,8 +93,13 @@ def _record(session, variant, *, assay_level="cdna", hgvs_assay_level=None): return record -def _link(session, record, allele, *, is_authoritative=False): - link = MappingRecordAllele(mapping_record_id=record.id, allele_id=allele.id, is_authoritative=is_authoritative) +def _link(session, record, allele, *, is_authoritative=False, projection_group=None): + link = MappingRecordAllele( + mapping_record_id=record.id, + allele_id=allele.id, + is_authoritative=is_authoritative, + projection_group=projection_group, + ) session.add(link) session.commit() return link @@ -143,8 +148,12 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): ) record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") - _link(session, record, measured, is_authoritative=True) + # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the + # protein apex is in no pair (group None). + _link(session, record, measured, is_authoritative=True, projection_group=0) + _link(session, record, genomic, projection_group=0) _link(session, record, protein) _vep(session, measured, "missense_variant") @@ -161,20 +170,33 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" # The alleles sidecar carries one identity per linked allele, keyed by digest: level + - # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation. - assert set(detail.alleles) == {"cdna-digest", "prot-digest"} + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + + # derivation + projection_of. + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest"} measured_identity = detail.alleles["cdna-digest"] assert measured_identity.level == "cdna" assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c assert measured_identity.clingen_allele_id == "CA123" assert measured_identity.relation is None # the defining allele has no relation to itself + # The measured allele is authoritative and pairs with its genomic projection sibling. + assert measured_identity.derivation == "authoritative" + assert measured_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. + assert genomic_identity.level == "genomic" + assert genomic_identity.derivation == "projection" + assert genomic_identity.projection_of == "cdna-digest" protein_identity = detail.alleles["prot-digest"] assert protein_identity.level == "protein" assert protein_identity.hgvs == "NP_000537.3:p.Ala406Thr" # coalesced from hgvs_p - # The protein member is a translation of the measured coding allele (member -> defining). + # The protein member is a translation of the measured coding allele (member -> defining)... assert protein_identity.relation == "translation_of" - # Annotations keyed by digest, covering both linked alleles; VEP rode in on the measured allele. - assert set(detail.annotations) == {"cdna-digest", "prot-digest"} + # ...and, on a nucleotide assay, is itself a deterministic projection (axes are independent) with no + # projection sibling (the apex is in no c/g pair). + assert protein_identity.derivation == "projection" + assert protein_identity.projection_of is None + # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest"} assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True @@ -183,15 +205,22 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): @pytest.mark.integration def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_score_set): - """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, and - the nt member `encodes` the defining protein allele.""" + """Mode 2 (protein measured): assay level is protein, target HGVS is the submitted p. string, the nt + members `encode` the defining protein allele, and the fanned-out c/g candidates are labelled + `candidate` (ambiguous) while still pairing to each other via projection_of. The protein apex is in + no pair (group None). The `relation` (structural) and `derivation` (provenance) axes are independent: + the coding candidate is `encodes` + `candidate` here, versus `translation_of` + `projection` for the + protein member of a nucleotide assay.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A", hgvs_pro="p.Ala406Thr") record = _record(session, variant, assay_level="protein", hgvs_assay_level="NP_000537.3:p.Ala406Thr") measured = _allele(session, "prot-digest", level="protein", clingen_allele_id="PA9") - coding = _allele(session, "cdna-digest", level="cdna") - _link(session, record, measured, is_authoritative=True) - _link(session, record, coding) + coding = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") + _link(session, record, measured, is_authoritative=True) # protein apex — group None + # One reverse-translation projection pair: a coding candidate paired with its genomic projection. + _link(session, record, coding, projection_group=0) + _link(session, record, genomic, projection_group=0) detail = get_variant_detail(session, variant) @@ -200,11 +229,43 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - # Defining protein allele has no relation to itself; the coding member encodes it. - assert detail.alleles["prot-digest"].level == "protein" - assert detail.alleles["prot-digest"].relation is None - assert detail.alleles["cdna-digest"].level == "cdna" - assert detail.alleles["cdna-digest"].relation == "encodes" + # Defining protein allele has no relation to itself, is authoritative, and is in no c/g pair. + prot = detail.alleles["prot-digest"] + assert prot.level == "protein" + assert prot.relation is None + assert prot.derivation == "authoritative" + assert prot.projection_of is None + # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` + # (provenance) — the two axes disagree by design — and pairs to its genomic projection. + coding_identity = detail.alleles["cdna-digest"] + assert coding_identity.relation == "encodes" + assert coding_identity.derivation == "candidate" + assert coding_identity.projection_of == "gen-digest" + genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.relation == "encodes" + assert genomic_identity.derivation == "candidate" + assert genomic_identity.projection_of == "cdna-digest" + + +@pytest.mark.integration +def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): + """Links written before reverse translation ran carry a NULL projection_group: derivation is still + computed (authoritative / projection from is_authoritative + assay level), but projection_of degrades + to null — nothing pairs until the data is re-processed.""" + score_set = setup_lib_db_with_score_set + variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") + record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") + measured = _allele(session, "cdna-digest", level="cdna", hgvs_c="NM_000546.6:c.1216G>A") + protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + _link(session, record, measured, is_authoritative=True) # no projection_group (pre-RT) + _link(session, record, protein) + + detail = get_variant_detail(session, variant) + + assert detail.alleles["cdna-digest"].derivation == "authoritative" + assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling + assert detail.alleles["prot-digest"].derivation == "projection" + assert detail.alleles["prot-digest"].projection_of is None @pytest.mark.integration diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 30e5af017..632c0fa15 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4562,7 +4562,8 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( def _seed_lean_mapping(session, variant_urn): """Give one variant a live coding-measured mapping record (with an assay-level HGVS) whose - authoritative allele carries a digest + ClinGen id + a live VEP consequence.""" + authoritative allele carries a digest + ClinGen id + a live VEP consequence, plus its canonical + genomic projection sibling (shared ``projection_group``) and the protein apex — the full triple.""" seed_mapping_record( session, variant_urn, @@ -4575,6 +4576,13 @@ def _seed_lean_mapping(session, variant_urn): is_authoritative=True, clingen_allele_id="CA123", vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-digest", + level="genomic", + hgvs_g="NC_000017.11:g.7676154C>T", + projection_group=0, ), AlleleSpec(digest="prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr"), ], @@ -4605,21 +4613,31 @@ def test_get_lean_variants(client, session, data_provider, data_files, setup_rou # Submitted HGVS (from the uploaded CSV), each with its parsed block riding alongside. assert mapped["hgvsNt"] == {"hgvs": "c.1A>T", "position": 1, "ref": "A", "alt": "T"} assert mapped["hgvsPro"] == {"hgvs": "p.Thr1Ser", "position": 1, "ref": "Thr", "alt": "Ser"} - # Mapped assay-level HGVS (reference frame) and the mapped protein representation. - assert mapped["assayLevelHgvs"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} - assert mapped["proteinLevelHgvs"] == { + # The mapped triple (reference frame): the measured slot named by assayLevel, its projection_group + # genomic sibling, and the protein apex. mapped.cdna is the search key even here (measured at cdna). + assert mapped["assayLevel"] == "cdna" + assert mapped["mapped"]["cdna"] == {"hgvs": "NM_000546.6:c.1216G>A", "position": 1216, "ref": "G", "alt": "A"} + assert mapped["mapped"]["genomic"] == { + "hgvs": "NC_000017.11:g.7676154C>T", + "position": 7676154, + "ref": "C", + "alt": "T", + } + assert mapped["mapped"]["protein"] == { "hgvs": "NP_000537.3:p.Ala406Thr", "position": 406, "ref": "Ala", "alt": "Thr", } - # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none). + # An unmapped variant keeps its submitted HGVS + score; the mapped fields are dropped (exclude_none), + # and the mapped triple serializes empty (no slots) since none is populated. unmapped = records[1] assert unmapped["score"] == 1.0 assert unmapped["hgvsNt"]["hgvs"] == "c.2C>T" - for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevelHgvs"): + for omitted in ("consequence", "clingenAlleleId", "assayLevelDigest", "assayLevel"): assert omitted not in unmapped + assert unmapped["mapped"] == {} def test_get_lean_variants_unknown_score_set_is_404(client, setup_router_db): diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 8bb1c3503..ea1c01086 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -53,7 +53,8 @@ def _post_mapped() -> dict: def _seed_mapping(session, variant_urn): """Give a variant a live coding-measured mapping record whose authoritative allele carries a - digest + ClinGen id + a live VEP consequence, plus a protein member — enough to build Cat-VRS.""" + digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared + projection_group), and a protein apex — enough to build Cat-VRS and exercise the projection axes.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -71,15 +72,25 @@ def _seed_mapping(session, variant_urn): clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A", ) + genomic = Allele( + vrs_digest="gen-digest", level="genomic", post_mapped=_post_mapped(), hgvs_g="NC_000017.11:g.7676154C>T" + ) protein = Allele( vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" ) - session.add_all([measured, protein]) + session.add_all([measured, genomic, protein]) session.commit() session.add_all( [ - MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True), + # The measured cdna link and its genomic projection share a projection_group; the protein + # apex is in no pair (group None). + MappingRecordAllele( + mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 + ), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 + ), MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), VepAlleleConsequence( allele_id=measured.id, @@ -120,9 +131,17 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + # Provenance axis (derivation) + the projection pairing (projectionOf), camelCased. + assert body["alleles"]["cdna-digest"]["derivation"] == "authoritative" + assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["derivation"] == "projection" + assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" + # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). + assert body["alleles"]["prot-digest"]["derivation"] == "projection" + assert "projectionOf" not in body["alleles"]["prot-digest"] assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 1a77e8e86..7bc33cf99 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -9,7 +9,12 @@ from datetime import timedelta from unittest.mock import MagicMock, patch -from variant_annotation.lib.translation.types import TranslationError, TranslationResult, WtCodonMode +from variant_annotation.lib.translation.types import ( + ProjectionPair, + TranslationError, + TranslationResult, + WtCodonMode, +) from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele @@ -60,8 +65,17 @@ def fake_construct(results_by_hgvs: dict, errors_by_hgvs: dict | None = None): The job correlates each TranslationResult/TranslationError back to its originating mapping record via object identity on ``result.input``, so the fake must echo the - exact VariantInput instances it was handed. Candidates are keyed by the input's + exact VariantInput instances it was handed. Results are keyed by the input's assay-level HGVS string. + + Each result spec is one of: + + * a list of ``(hgvs_c, hgvs_g)`` pairs — one ProjectionPair each. + ``hgvs_g`` may be ``None`` for a projection-failed, coding-only projection pair. + * a ``(pairs, hgvs_p)`` tuple pairing that list with a shared protein apex. + + Each pair becomes one ProjectionPair, mirroring the position-aligned + coding↔genomic rows the real reverse-translate CLI emits (with ``--one-row-per-input``). """ errors_by_hgvs = errors_by_hgvs or {} @@ -71,14 +85,13 @@ def _construct(inputs, *, transcripts, coordinates, config): if inp.hgvs in errors_by_hgvs: errors.append(TranslationError(input=inp, error=errors_by_hgvs[inp.hgvs])) elif inp.hgvs in results_by_hgvs: - entry = results_by_hgvs[inp.hgvs] - c_candidates, g_candidates = entry[0], entry[1] - hgvs_p = entry[2] if len(entry) > 2 else None + spec = results_by_hgvs[inp.hgvs] + # A bare list is the candidate pairs; a tuple pairs them with a protein apex. + pairs, hgvs_p = (spec[0], spec[1]) if isinstance(spec, tuple) else (spec, None) results.append( TranslationResult( input=inp, - hgvs_c_candidates=list(c_candidates), - hgvs_g_candidates=list(g_candidates), + projection_pairs=[ProjectionPair(hgvs_c=hgvs_c, hgvs_g=hgvs_g) for hgvs_c, hgvs_g in pairs], hgvs_p=hgvs_p, ) ) @@ -163,6 +176,34 @@ def _non_authoritative_links(session): return session.query(MappingRecordAllele).filter(MappingRecordAllele.is_authoritative.is_(False)).all() +def _live_links(session, record_id): + """Every live (valid_to IS NULL) link for a mapping record, authoritative or derived.""" + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.valid_to.is_(None), + ) + .all() + ) + + +def _authoritative_link(session, record_id): + return ( + session.query(MappingRecordAllele) + .filter( + MappingRecordAllele.mapping_record_id == record_id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .one() + ) + + +def _record_for(session, variant_id): + return session.query(MappingRecord).filter(MappingRecord.variant_id == variant_id).one() + + @pytest.mark.unit class TestBuildTranslationConfig: """Unit tests for _build_translation_config — the optional translation_config job param.""" @@ -241,7 +282,8 @@ async def test_creates_genomic_and_coding_candidate_alleles( sample_score_set, ): """A mapped variant expands into one non-authoritative allele per equivalence-class - candidate, each linked to the originating mapping record.""" + candidate, each linked to the originating mapping record. The two members of a single + projection pair (coding + genomic projection) share one projection_group.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -256,7 +298,7 @@ async def test_creates_genomic_and_coding_candidate_alleles( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) with ( @@ -286,12 +328,20 @@ async def test_creates_genomic_and_coding_candidate_alleles( assert coding_allele.transcript == "NM_000001.1" # The candidate alleles are linked to the same mapping record as the authoritative allele. - mapping_record = session.query(MappingRecord).filter(MappingRecord.variant_id == variant.id).one() + mapping_record = _record_for(session, variant.id) assert {link.mapping_record_id for link in non_auth_links} == {mapping_record.id} + # Both members of the single projection pair carry the same (first, index 0) group id — the + # coding↔genomic pairing survives the flatten that Phase B exists to stop. + assert {link.projection_group for link in non_auth_links} == {0} + events = _cross_level_events(session, sample_score_set.id) assert len(events) == 1 assert events[0].disposition == "present" + # The event metadata carries the paired projection pair shape, not the removed flat lists. + assert events[0].event_metadata["candidates"] == [ + {"hgvs_c": c_candidate, "hgvs_g": g_candidate, "variant_type": None} + ] async def test_transcript_is_queryable_via_derived_expression( self, @@ -319,7 +369,7 @@ async def test_transcript_is_queryable_via_derived_expression( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) with ( @@ -346,7 +396,8 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( ): """The deterministic protein consequence (``result.hgvs_p``) is emitted as the protein-level member of the equivalence set -- a non-authoritative ``level=protein`` - allele linked to the record, not dropped.""" + allele linked to the record, not dropped. As the apex shared across all projection pairs it + belongs to no projection pair, so its link's projection_group is NULL.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -364,7 +415,7 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( # c_to_p emits a predicted consequence in parens; the job strips them before use. p_consequence = "NP_000001.1:p.(Met1Val)" p_stripped = "NP_000001.1:p.Met1Val" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate], p_consequence)}) + construct = fake_construct({assay_hgvs: ([(c_candidate, g_candidate)], p_consequence)}) translate = fake_translate( { g_candidate: "ga4gh:VA.genomic", @@ -382,8 +433,14 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() assert protein_allele.level == AnnotationLayer.protein.value assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens - # Linked to the record as a non-authoritative member of the equivalence set. - assert protein_allele.id in {link.allele_id for link in _non_authoritative_links(session)} + + # Linked to the record as a non-authoritative member of the equivalence set, but grouped + # with no pair (the apex is shared across every projection pair). + protein_link = next(link for link in _non_authoritative_links(session) if link.allele_id == protein_allele.id) + assert protein_link.projection_group is None + # The coding/genomic pair is still grouped together (group 0). + pair_links = [link for link in _non_authoritative_links(session) if link.allele_id != protein_allele.id] + assert {link.projection_group for link in pair_links} == {0} async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( self, @@ -395,8 +452,9 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( sample_independent_reverse_translation_run, sample_score_set, ): - """A bracketed genomic candidate (non-adjacent codon components) is translated into a - CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest.""" + """A bracketed genomic projection (non-adjacent codon components) is translated into a + CisPhasedBlock and persisted like any other candidate allele, keyed by its CPB digest, and + grouped with its coding partner in the projection pair.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -409,10 +467,11 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" block_candidate = "NC_000001.11:g.[1000A>G;1002T>C]" - construct = fake_construct({assay_hgvs: ([], [block_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, block_candidate)]}) translate = fake_translate( - {block_candidate: "ga4gh:CPB.block"}, + {c_candidate: "ga4gh:VA.coding", block_candidate: "ga4gh:CPB.block"}, type_by_hgvs={block_candidate: "CisPhasedBlock"}, ) @@ -423,7 +482,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() assert block_allele.level == AnnotationLayer.genomic.value @@ -432,7 +491,8 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( assert block_allele.transcript == "NC_000001.11" assert block_allele.post_mapped == {"type": "CisPhasedBlock", "id": "ga4gh:CPB.block"} - assert len(_non_authoritative_links(session)) == 1 + # The block (genomic projection) and its coding partner are one projection pair -> one group. + assert {link.projection_group for link in _non_authoritative_links(session)} == {0} async def test_duplicate_candidate_digests_are_deduped_per_record( self, @@ -444,8 +504,9 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( sample_independent_reverse_translation_run, sample_score_set, ): - """When a coding and genomic candidate resolve to the same VRS digest, only one - allele/link is written for that mapping record (per the library's dedup contract).""" + """When the coding and genomic members of a projection pair resolve to the same VRS digest, + only one allele/link is written for that mapping record (per the library's dedup contract). + The surviving link still carries the pair's group.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -460,8 +521,8 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) - # Both candidates collapse to the same VRS digest. + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + # Both members of the projection pair collapse to the same VRS digest. translate = fake_translate({g_candidate: "ga4gh:VA.shared", c_candidate: "ga4gh:VA.shared"}) with ( @@ -472,10 +533,12 @@ async def test_duplicate_candidate_digests_are_deduped_per_record( assert result.status == JobStatus.SUCCEEDED assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - assert len(_non_authoritative_links(session)) == 1 + non_auth_links = _non_authoritative_links(session) + assert len(non_auth_links) == 1 + assert non_auth_links[0].projection_group == 0 assert session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.shared").count() == 1 - async def test_candidate_equal_to_authoritative_allele_is_not_relinked( + async def test_authoritative_allele_is_folded_into_its_projection_group( self, session, with_independent_processing_runs, @@ -485,10 +548,11 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( sample_independent_reverse_translation_run, sample_score_set, ): - """A derived candidate whose digest equals the record's authoritative (assay-measured) - allele is not linked again — the record already links that allele authoritatively, and a - derived duplicate would surface the measured variant twice. No new link and no new allele; - an empty derived set is still a success with nothing to add.""" + """Nucleotide assay: the measured allele appears among the RT candidates (the library makes + no exclusions) and already has a live authoritative link (group NULL). Rather than insert a + forbidden derived duplicate, the job folds the pair's group onto the existing + authoritative link. That group then resolves to {canonical coding, canonical genomic}: the + authoritative allele plus its sibling projection derived in the same projection pair.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -500,16 +564,16 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) - # Exactly one allele exists after mapping: the authoritative post-mapped allele. + # Exactly one allele exists after mapping: the authoritative (measured) allele. assert session.query(Allele).count() == 1 - authoritative_allele = session.query(Allele).one() - assert authoritative_allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER assay_hgvs = "NM_000000.1:c.1A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [])}) - # The candidate resolves to the same digest as the authoritative allele. - translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER}) + g_sibling = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_candidate, g_sibling)]}) + # The coding member IS the measured allele (folds in); the genomic member is its projection. + translate = fake_translate({c_candidate: TEST_GA4GH_IDENTIFIER, g_sibling: "ga4gh:VA.genomic"}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -518,14 +582,198 @@ async def test_candidate_equal_to_authoritative_allele_is_not_relinked( result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 0} + # Only the genomic sibling is a newly created derived allele; the coding member folds into + # the existing authoritative link rather than creating a second link/allele. + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} - # The authoritative allele was reused, not duplicated. - assert session.query(Allele).count() == 1 - # No derived link was added: the candidate equals the record's authoritative allele. - assert _non_authoritative_links(session) == [] + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert len(derived_links) == 1 + + # The authoritative link was folded into the pair's group, and the sibling derived + # link shares it -- the pairing is resolvable from the authoritative allele's group. + assert auth_link.projection_group == 0 + assert derived_links[0].projection_group == 0 + + # group 0 resolves to {canonical c (authoritative), canonical g (projection)}: two live + # links, one authoritative and one derived. + group_links = [link for link in _live_links(session, record.id) if link.projection_group == 0] + assert len(group_links) == 2 + assert {link.is_authoritative for link in group_links} == {True, False} + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.genomic" + + async def test_authoritative_fold_in_when_measured_at_genomic_level( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """The fold-in is symmetric in the measured level: when the genomic member of a projection pair + equals the record's authoritative allele, its group folds onto the authoritative link and + the coding sibling becomes the derived member of the same group.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NC_000001.11:g.1000A>G", + data={}, + ) + session.add(variant) + session.commit() + # Genomic assay layer + cdna identity mapping so RT resolves a coding transcript. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"g", "c"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NC_000001.11:g.1000A>G" + c_sibling = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.1000A>G" + construct = fake_construct({assay_hgvs: [(c_sibling, g_candidate)]}) + # The genomic member IS the measured allele (folds in); the coding member is its projection. + translate = fake_translate({c_sibling: "ga4gh:VA.coding", g_candidate: TEST_GA4GH_IDENTIFIER}) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + + record = _record_for(session, variant.id) + auth_link = _authoritative_link(session, record.id) + derived_links = _non_authoritative_links(session) + assert auth_link.projection_group == 0 + assert [link.projection_group for link in derived_links] == [0] + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + assert derived_links[0].allele.vrs_digest == "ga4gh:VA.coding" + + async def test_projection_failed_candidate_is_a_one_member_group( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """A projection pair whose genomic projection failed (hgvs_g is None) yields a well-formed + one-member (coding-only) group -- no crash, no desync, and the group holds just the coding + link. This is the case the old two-list shape silently dropped.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_nt="NM_000000.1:c.1A>G", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) + + assay_hgvs = "NM_000000.1:c.1A>G" + c_paired = "NM_000001.1:c.5A>G" + g_paired = "NC_000001.11:g.1000A>G" + c_orphan = "NM_000001.1:c.9A>T" # projection failed: no genomic partner + construct = fake_construct({assay_hgvs: [(c_paired, g_paired), (c_orphan, None)]}) + translate = fake_translate( + { + c_paired: "ga4gh:VA.coding0", + g_paired: "ga4gh:VA.genomic0", + c_orphan: "ga4gh:VA.coding1", + } + ) + + with ( + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 3} - async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( + non_auth_links = _non_authoritative_links(session) + groups: dict[int, set[str]] = {} + for link in non_auth_links: + groups.setdefault(link.projection_group, set()).add(link.allele.vrs_digest) + + # Group 0 is the full projection pair; group 1 is the projection-failed coding-only member. + assert groups == { + 0: {"ga4gh:VA.coding0", "ga4gh:VA.genomic0"}, + 1: {"ga4gh:VA.coding1"}, + } + + async def test_protein_assay_leaves_authoritative_group_null_over_a_fan_out( + self, + session, + with_independent_processing_runs, + with_reverse_translation_run, + mock_worker_ctx, + sample_independent_variant_mapping_run, + sample_independent_reverse_translation_run, + sample_score_set, + ): + """Protein assay: the authoritative allele is the protein itself, which appears in no + coding/genomic projection pair, so nothing folds in and its link's group stays NULL. The + ambiguous fan-out becomes N distinct groups, each a coding/genomic pair.""" + variant = Variant( + score_set_id=sample_score_set.id, + urn="variant:1", + hgvs_pro="NP_000000.1:p.Met1Val", + data={}, + ) + session.add(variant) + session.commit() + # Protein-only mapping: the authoritative allele is the protein allele. + await _map_variants( + session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set, with_layers={"p"} + ) + assert session.query(Allele).one().vrs_digest == TEST_GA4GH_IDENTIFIER + + assay_hgvs = "NP_000000.1:p.Met1Val" + # A degenerate protein consequence -> three coding/genomic projection pairs (the fan-out). + pairs = [ + ("NM_000001.1:c.3A>G", "NC_000001.11:g.100A>G"), + ("NM_000001.1:c.3A>C", "NC_000001.11:g.100A>C"), + ("NM_000001.1:c.3A>T", "NC_000001.11:g.100A>T"), + ] + # Protein assay: no protein apex is re-emitted (result.hgvs_p is None) -- the protein + # allele is already authoritative. + construct = fake_construct({assay_hgvs: pairs}) + translate = fake_translate({hgvs: f"ga4gh:VA.{hgvs}" for pair in pairs for hgvs in pair}) + + with ( + patch(f"{RT_MODULE}._coding_transcripts_for_proteins", lambda accessions: {"NP_000000.1": "NM_000001.1"}), + patch(f"{RT_MODULE}.construct_equivalent_variants", construct), + patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), + ): + result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) + + assert result.status == JobStatus.SUCCEEDED + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 6} + + record = _record_for(session, variant.id) + # The protein authoritative allele is not part of any pair: its group stays NULL. + auth_link = _authoritative_link(session, record.id) + assert auth_link.projection_group is None + assert auth_link.allele.vrs_digest == TEST_GA4GH_IDENTIFIER + + # Three distinct groups, each a coding/genomic pair (two members). + groups: dict[int, int] = {} + for link in _non_authoritative_links(session): + groups[link.projection_group] = groups.get(link.projection_group, 0) + 1 + assert groups == {0: 2, 1: 2, 2: 2} + + async def test_independent_rerun_retires_prior_links_and_regroups( self, session, with_independent_processing_runs, @@ -536,9 +784,11 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( sample_score_set, ): """An independent second run is idempotent: it retires the prior derived links (closing - valid_to) and writes fresh live ones, so the set of *current* non-authoritative links is - stable while the superseded links are retained as history — the partial unique index on - live links is never violated and no alleles are duplicated.""" + valid_to) and writes fresh live ones (re-grouped from scratch), so the set of *current* + non-authoritative links is stable while the superseded links are retained as history -- + the partial unique index on live links is never violated and no alleles are duplicated. + projection_group is regenerated per run; it is a within-record grouping key, not a stable + identity across re-maps.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -553,10 +803,10 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( assay_hgvs = "NM_000000.1:c.1A>G" g_candidate = "NC_000001.11:g.1000A>G" c_candidate = "NM_000001.1:c.5A>G" - construct = fake_construct({assay_hgvs: ([c_candidate], [g_candidate])}) + construct = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) translate = fake_translate({g_candidate: "ga4gh:VA.genomic", c_candidate: "ga4gh:VA.coding"}) - # First run: two live derived links. + # First run: two live derived links, both in group 0. with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), patch(f"{RT_MODULE}.translate_hgvs_to_variation", translate), @@ -567,6 +817,7 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( first_links = _non_authoritative_links(session) assert len(first_links) == 2 assert all(link.valid_to is None for link in first_links) + assert {link.projection_group for link in first_links} == {0} # Second, independent run with identical inputs. rerun = JobRun( @@ -592,13 +843,14 @@ async def test_independent_rerun_retires_prior_links_and_keeps_current_stable( live = [link for link in links if link.valid_to is None] retired = [link for link in links if link.valid_to is not None] - # The live set is stable (still exactly the two candidates) and the prior run's links are - # retained as closed history rather than deleted. + # The live set is stable (still exactly the two candidates, re-grouped into group 0) and the + # prior run's links are retained as closed history rather than deleted. assert len(live) == 2 assert len(retired) == 2 + assert {link.projection_group for link in live} == {0} # Gap-free handoff: the prior links closed at exactly the instant the new links opened, under - # one supersession timestamp — so a point-in-time query never lands in a hole between runs, + # one supersession timestamp -- so a point-in-time query never lands in a hole between runs, # even though the translation loop ran (and could have committed) between retire and insert. assert {link.valid_to for link in retired} == {link.valid_from for link in live} assert len({link.valid_from for link in live}) == 1 @@ -683,12 +935,12 @@ async def test_partial_success_succeeds_with_mixed_annotations( session.commit() await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) - g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" construct = fake_construct( - {"NM_000000.1:c.1A>G": ([], [g_candidate])}, + {"NM_000000.1:c.1A>G": [(c_candidate, None)]}, errors_by_hgvs={"NM_000000.1:c.2G>T": "no consequence"}, ) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) with ( patch(f"{RT_MODULE}.construct_equivalent_variants", construct), @@ -713,8 +965,8 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata sample_independent_reverse_translation_run, sample_score_set, ): - """One candidate translating and another failing VRS translation leaves the variant a - SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" + """One member of a projection pair translating and the other failing VRS translation leaves the + variant a SUCCESS (one allele created) while retaining the dropped candidate in metadata.""" variant = Variant( score_set_id=sample_score_set.id, urn="variant:1", @@ -727,12 +979,12 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" - good_candidate = "NC_000001.11:g.1000A>G" - bad_candidate = "NC_000001.11:g.999A>T" - construct = fake_construct({assay_hgvs: ([], [good_candidate, bad_candidate])}) + good_coding = "NM_000001.1:c.5A>G" + bad_genomic = "NC_000001.11:g.999A>T" + construct = fake_construct({assay_hgvs: [(good_coding, bad_genomic)]}) translate = fake_translate( - {good_candidate: "ga4gh:VA.genomic"}, - errors_by_hgvs={bad_candidate: "untranslatable form"}, + {good_coding: "ga4gh:VA.coding"}, + errors_by_hgvs={bad_genomic: "untranslatable form"}, ) with ( @@ -748,7 +1000,7 @@ async def test_partial_candidate_translation_failure_keeps_success_with_metadata events = _cross_level_events(session, sample_score_set.id, disposition="present") assert len(events) == 1 failed_candidates = events[0].event_metadata["failed_candidates"] - assert failed_candidates == [{"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"}] + assert failed_candidates == [{"hgvs": bad_genomic, "level": "genomic", "error": "untranslatable form"}] async def test_all_candidates_failing_translation_marks_variant_failed( self, @@ -774,8 +1026,8 @@ async def test_all_candidates_failing_translation_marks_variant_failed( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" - bad_candidate = "NC_000001.11:g.999A>T" - construct = fake_construct({assay_hgvs: ([], [bad_candidate])}) + bad_candidate = "NM_000001.1:c.9A>T" # a coding-only projection pair whose sole member fails + construct = fake_construct({assay_hgvs: [(bad_candidate, None)]}) translate = fake_translate({}, errors_by_hgvs={bad_candidate: "untranslatable form"}) with ( @@ -794,7 +1046,7 @@ async def test_all_candidates_failing_translation_marks_variant_failed( metadata = failed_events[0].event_metadata assert metadata["hgvs_input"] == assay_hgvs assert metadata["failed_candidates"] == [ - {"hgvs": bad_candidate, "level": "genomic", "error": "untranslatable form"} + {"hgvs": bad_candidate, "level": "cdna", "error": "untranslatable form"} ] async def test_unresolved_transcript_is_tallied_as_skip_but_recorded_as_failed( @@ -876,15 +1128,21 @@ async def test_genomic_accession_coding_target_is_reverse_translated( ) assay_hgvs = "NC_000001.11:g.1000A>G" - g_candidate = "NC_000001.11:g.1000A>G" + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" with ( - patch(f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: ([], [g_candidate])})), - patch(f"{RT_MODULE}.translate_hgvs_to_variation", fake_translate({g_candidate: "ga4gh:VA.genomic"})), + patch( + f"{RT_MODULE}.construct_equivalent_variants", fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + ), + patch( + f"{RT_MODULE}.translate_hgvs_to_variation", + fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}), + ), ): result = await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) assert result.status == JobStatus.SUCCEEDED - assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 1} + assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} events = _cross_level_events(session, sample_score_set.id) assert events and all(e.disposition == "present" for e in events) @@ -944,8 +1202,9 @@ async def test_uses_latest_cdna_row_within_the_run( session.commit() assay_hgvs = "NC_000001.11:g.1000A>G" - g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + c_candidate = "NM_000001.1:c.5A>G" + g_candidate = "NC_000001.11:g.2000A>G" + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) captured: dict = {} def capturing_construct(inputs, *, transcripts, coordinates, config): @@ -1116,9 +1375,10 @@ async def test_translation_config_param_overrides_job_defaults( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) captured: dict = {} @@ -1168,9 +1428,10 @@ async def test_translation_config_defaults_when_param_absent( await _map_variants(session, mock_worker_ctx, sample_independent_variant_mapping_run, sample_score_set) assay_hgvs = "NM_000000.1:c.1A>G" + c_candidate = "NM_000001.1:c.5A>G" g_candidate = "NC_000001.11:g.1000A>G" - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + delegate = fake_construct({assay_hgvs: [(c_candidate, g_candidate)]}) + translate = fake_translate({c_candidate: "ga4gh:VA.coding", g_candidate: "ga4gh:VA.genomic"}) captured: dict = {} @@ -1215,13 +1476,13 @@ async def test_protein_level_transcript_resolved_via_uta( ) assay_hgvs = "NP_000000.1:p.Met1Val" - g_candidate = "NC_000001.11:g.1000A>G" - translate = fake_translate({g_candidate: "ga4gh:VA.genomic"}) + c_candidate = "NM_000111.1:c.3A>G" + translate = fake_translate({c_candidate: "ga4gh:VA.coding"}) # Capture the transcript hint each VariantInput was built with, then delegate to the # normal stub for the translation result. captured_hints: dict[str, str | None] = {} - delegate = fake_construct({assay_hgvs: ([], [g_candidate])}) + delegate = fake_construct({assay_hgvs: [(c_candidate, None)]}) def capturing_construct(inputs, *, transcripts, coordinates, config): captured_hints.update({inp.hgvs: inp.transcript for inp in inputs}) From feb9c55eec41dba080514f7ff4810396a206bb9c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 9 Jul 2026 10:31:13 -0700 Subject: [PATCH 68/93] feat(allele-measurements): add include_nucleotide_siblings flag and remove legacy lookup endpoint - Add `include_nucleotide_siblings` parameter to `get_allele_measurements` and expose it as a query param on `GET /clingen-alleles/{id}/measurements`; for a CA query, widens the equivalence class through the queried change's protein consequence to surface sibling nt variants encoding the same amino-acid change (relationship=nucleotide_encoding) - Fix relationship labeling to key off measured-allele level rather than entry level, correcting the sibling-nt branch so sibling nt records get nucleotide_encoding instead of protein_consequence - Remove `POST /variants/clingen-allele-id-lookups` and its associated view models (ClingenAlleleIdVariantLookupsRequest, ClingenAlleleVariants, ClingenAlleleIdVariantLookupResponse, VariantEffectMeasurementWithShortScoreSet) --- src/mavedb/lib/allele_measurements.py | 47 ++- src/mavedb/routers/clingen_alleles.py | 12 + src/mavedb/routers/variants.py | 399 +------------------------- src/mavedb/view_models/variant.py | 39 +-- tests/lib/test_allele_measurements.py | 33 +++ tests/routers/test_clingen_allele.py | 36 +++ 6 files changed, 123 insertions(+), 443 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index eabb414f5..a3048cb83 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -8,8 +8,9 @@ consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein -is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. See -design §7.5. +is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. +``include_nucleotide_siblings`` opts out of that asymmetry for discovery surfaces — see +:func:`get_allele_measurements`. Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels detail of one *selected* measurement. This list is the *cross-record* union — who else measured @@ -53,9 +54,13 @@ class MeasurementRelationship(str, Enum): to the query. """ - direct = "direct" # the measurement was assayed *at* this allele - protein_consequence = "protein_consequence" # CA query, protein measurement of consequence(N) - nucleotide_encoding = "nucleotide_encoding" # PA query, nt measurement encoding P + # the measurement was assayed *at* this allele + direct = "direct" + # CA query, protein measurement of consequence(N) + protein_consequence = "protein_consequence" + # A nt measurement encoding the queried change's protein consequence: a PA query's encodings of P, or — + # only under ``include_nucleotide_siblings`` — a CA query's sibling nt changes that share consequence(N). + nucleotide_encoding = "nucleotide_encoding" @dataclass(frozen=True) @@ -148,10 +153,16 @@ def get_allele_measurements( *, user_data: Optional[UserData], include_superseded: bool = False, + include_nucleotide_siblings: bool = False, as_of: Optional[datetime] = None, ) -> list[AlleleMeasurement]: """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns ``[]`` when the id resolves to no allele or no live record. + + ``include_nucleotide_siblings`` (a ``CA`` entry only; a no-op for ``PA``) widens the class through the + queried change's protein consequence to also surface the *sibling* nt changes — other nucleotide + variants that encode the same amino-acid change and were themselves assayed at the nucleotide level + (``relationship=nucleotide_encoding``). """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() if not anchor: @@ -160,6 +171,23 @@ def get_allele_measurements( anchor_ids = [row.id for row in anchor] entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence + # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record + # union below also reaches every record encoding that consequence. a PA query already anchors on the protein, so + # this is exactly what makes a CA+siblings query behave like one. (This query would be redundant for a PA entry). + if include_nucleotide_siblings and not entry_is_protein: + anchor_link = aliased(MappingRecordAllele) + anchor_ids += db.scalars( + select(MappingRecordAllele.allele_id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == AnnotationLayer.protein.value) + .distinct() + ).all() + record_ids = db.scalars( select(MappingRecordAllele.mapping_record_id) .where(MappingRecordAllele.allele_id.in_(anchor_ids)) @@ -207,12 +235,15 @@ def get_allele_measurements( if not is_current and not include_superseded: continue + # Label by the measured level, not the entry level: direct when the measured allele *is* the query, + # else protein→protein_consequence / nucleotide→nucleotide_encoding. Without siblings the sibling-nt + # branch is unreachable (a record links the anchor only via itself or its protein consequence). if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct - elif entry_is_protein: - relationship = MeasurementRelationship.nucleotide_encoding - else: + elif measured_allele.level == AnnotationLayer.protein.value: relationship = MeasurementRelationship.protein_consequence + else: + relationship = MeasurementRelationship.nucleotide_encoding assay_level = record.assay_level measurement = AlleleMeasurement( diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py index b3b626c02..64c6c1544 100644 --- a/src/mavedb/routers/clingen_alleles.py +++ b/src/mavedb/routers/clingen_alleles.py @@ -50,6 +50,16 @@ def get_clingen_allele_measurements( "measurements are a deliberate power-user / citation path, never surfaced by discovery." ), ), + include_nucleotide_siblings: bool = Query( + default=False, + description=( + "For a nucleotide (CA) query only: also return sibling nucleotide changes — other DNA " + "variants encoding the same protein consequence that were themselves assayed at the " + "nucleotide level (relationship 'nucleotide_encoding'). Default false: the variant page " + "anchors strictly on the queried allele. Discovery surfaces (search) set it to surface all " + "evidence bearing on the consequence. No-op for a protein (PA) query." + ), + ), as_of: Optional[datetime] = Query( default=None, description=( @@ -72,6 +82,7 @@ def get_clingen_allele_measurements( "requested_resource": clingen_allele_id, "as_of": as_of, "include_superseded": include_superseded, + "include_nucleotide_siblings": include_nucleotide_siblings, } ) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" @@ -81,5 +92,6 @@ def get_clingen_allele_measurements( clingen_allele_id, user_data=user_data, include_superseded=include_superseded, + include_nucleotide_siblings=include_nucleotide_siblings, as_of=as_of, ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 66f302df6..d7cfdfc15 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,6 +1,4 @@ -import itertools import logging -import re from datetime import datetime from typing import Optional @@ -8,8 +6,7 @@ from fastapi.exceptions import HTTPException from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session, joinedload -from sqlalchemy.sql import or_ +from sqlalchemy.orm import Session from mavedb import deps from mavedb.lib.authentication import get_current_user @@ -18,20 +15,12 @@ from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.variant_detail import get_variant_detail -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.variant_translation import VariantTranslation from mavedb.routers.shared import ( ACCESS_CONTROL_ERROR_RESPONSES, - BASE_400_RESPONSE, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models.variant import ( - ClingenAlleleIdVariantLookupResponse, - ClingenAlleleIdVariantLookupsRequest, -) from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -51,392 +40,6 @@ } -@router.post( - "/variants/clingen-allele-id-lookups", - status_code=200, - response_model=list[ClingenAlleleIdVariantLookupResponse], - responses={ - **BASE_400_RESPONSE, - **ACCESS_CONTROL_ERROR_RESPONSES, - }, - summary="Lookup variants by ClinGen Allele IDs", -) -def lookup_variants( - *, - request: ClingenAlleleIdVariantLookupsRequest, - db: Session = Depends(deps.get_db), - user_data: UserData = Depends(get_current_user), -): - """ - Lookup variants by ClinGen Allele IDs. - """ - save_to_logging_context({"requested_resource": "clingen-allele-id-lookups"}) - save_to_logging_context({"clingen_allele_ids_to_lookup": request.clingen_allele_ids}) - logger.debug(msg="Looking up variants by Clingen Allele IDs", extra=logging_context()) - - # sort multi-variant components lexicographically, as they are in the database - request.clingen_allele_ids = [",".join(sorted(allele_id.split(","))) for allele_id in request.clingen_allele_ids] - exact_match_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(request.clingen_allele_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - save_to_logging_context({"num_variants_matching_clingen_allele_ids": len(exact_match_variants)}) - logger.debug(msg="Found variants with exactly matching ClinGen Allele IDs", extra=logging_context()) - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - - variants_by_allele_id: dict[str, dict] = { - allele_id: { - "clingen_allele_id": allele_id, - "exact_match": {"clingen_allele_id": allele_id, "variant_effect_measurements": []}, - "equivalent_nt": [], - "equivalent_aa": [], - } - for allele_id in request.clingen_allele_ids - } - for variant, allele_id in exact_match_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - num_variants_matching_clingen_allele_ids_and_permitted += 1 - variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"].append(variant) - - save_to_logging_context( - {"clingen_allele_ids_with_permitted_variants": num_variants_matching_clingen_allele_ids_and_permitted} - ) - - for allele_id in request.clingen_allele_ids: - if not variants_by_allele_id[allele_id]["exact_match"]["variant_effect_measurements"]: - variants_by_allele_id[allele_id]["exact_match"] = None - - for allele_id in request.clingen_allele_ids: - # validate and determine whether multi-variant - # NOTE: assuming we never have more than 2 components in a multi-variant - single_clingen_allele_id_re = r"^[CP]A\d+$" - multi_clingen_allele_id_re = r"^[CP]A\d+,[CP]A\d+$" - single_or_multi_clingen_allele_id_re = r"^[CP]A\d+(,[CP]A\d+)?$" - - if re.fullmatch(single_or_multi_clingen_allele_id_re, allele_id) is None: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}'", - ) - - if re.fullmatch(single_clingen_allele_id_re, allele_id): - if allele_id.startswith("PA"): - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where(VariantTranslation.aa_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == allele_id, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_aa_clingen_ids = [ - var_translation.aa_clingen_id - for var_translation in related_clingen_ids - if var_translation.aa_clingen_id != allele_id - ] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_clingen_ids = [var_translation.nt_clingen_id for var_translation in related_clingen_ids] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - elif allele_id.startswith("CA"): - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where(VariantTranslation.nt_clingen_id == allele_id) - ) - .scalars() - .all() - ) - related_clingen_ids = ( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - related_aa_clingen_ids = [var_translation.aa_clingen_id for var_translation in related_clingen_ids] - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_aa_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - # exclude requested clingen allele id from "related_clingen_ids" to avoid duplicates - related_nt_clingen_ids = [ - var_translation.nt_clingen_id - for var_translation in related_clingen_ids - if var_translation.nt_clingen_id != allele_id - ] - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(related_nt_clingen_ids)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - num_variants_matching_clingen_allele_ids_and_permitted = 0 - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - elif re.fullmatch(multi_clingen_allele_id_re, allele_id): - # validate each component allele id - already done via re - # allow more than 2 components? - # full matches are determined the same exact way as single variant (done above already) - # equivalent aa/nt: if all components of variant are represented by equivalent or exact match, but not all exact matches because those are already represented above - # so basically go through variant translations table and create every possible combo of components except for full exact match, - # then search the db for those combos - - allele_id_components = allele_id.split(",") - if all(component.startswith("PA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.nt_clingen_id).where( - VariantTranslation.aa_clingen_id == component - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.aa_clingen_id == component, - VariantTranslation.nt_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[0]) - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - aa_clingen_id_first_allele.append(allele_id_components[1]) - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - elif all(component.startswith("CA") for component in allele_id_components): - related_clingen_id_components = [] # list of lists, one list for each component - for component in allele_id_components: - subquery = ( - db.execute( - select(VariantTranslation.aa_clingen_id).where( - VariantTranslation.nt_clingen_id == allele_id - ) - ) - .scalars() - .all() - ) - related_clingen_id_components.append( - db.execute( - select(VariantTranslation).where( - or_( - VariantTranslation.nt_clingen_id == allele_id, - VariantTranslation.aa_clingen_id.in_(subquery), - ) - ) - ) - .scalars() - .all() - ) - # create every possible combination of those two, including the original components, except for the version with both original components - # assuming that aa variants should always be paired with aa variants, and nt variants with nt variants, - # create lists of just the aa variants, and lists of just the nt variants. - # assuming only 2 components for any multivariant in db. - aa_clingen_id_first_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[0] - ] - aa_clingen_id_second_allele = [ - translation.aa_clingen_id for translation in related_clingen_id_components[1] - ] - related_aa_clingen_id_combinations = list( - itertools.product(aa_clingen_id_first_allele, aa_clingen_id_second_allele) - ) - # turn each inner list into a single string and sort each pair lexicographically, since this is how they are stored in the db - joined_related_aa_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_aa_clingen_id_combinations - ] - - related_nt_clingen_id_first_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[0] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[0]) - related_nt_clingen_id_second_allele = [ - translation.nt_clingen_id for translation in related_clingen_id_components[1] - ] - # original component should also be included, even if it is not in the variant translations table - related_nt_clingen_id_first_allele.append(allele_id_components[1]) - related_nt_clingen_id_combinations = list( - itertools.product(related_nt_clingen_id_first_allele, related_nt_clingen_id_second_allele) - ) - joined_related_nt_clingen_id_combinations = [ - ",".join(sorted(list(combination))) # type: ignore - for combination in related_nt_clingen_id_combinations - ] - # query the db for variants with these combinations - related_aa_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_aa_clingen_id_combinations)) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - related_nt_variants = db.execute( - select(Variant, MappedVariant.clingen_allele_id) - .join(MappedVariant) - .options(joinedload(Variant.score_set).joinedload(ScoreSet.experiment)) - .where(MappedVariant.clingen_allele_id.in_(joined_related_nt_clingen_id_combinations)) - .where(MappedVariant.clingen_allele_id != allele_id) - .where(MappedVariant.current == True) # noqa: E712 - ).all() - - else: - raise HTTPException( - status_code=400, - detail=f"Invalid Clingen Allele ID '{allele_id}': all components must be either PA or CA", - ) - - equivalent_aa_variants = {} - for variant, related_allele_id in related_aa_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_aa_variants: - equivalent_aa_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_aa_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_aa"] = [ - equivalent_aa_variants[related_allele_id] for related_allele_id in equivalent_aa_variants - ] - - equivalent_nt_variants = {} - for variant, related_allele_id in related_nt_variants: - if has_permission(user_data, variant.score_set, Action.READ).permitted: - if related_allele_id not in equivalent_nt_variants: - equivalent_nt_variants[related_allele_id] = { - "clingen_allele_id": related_allele_id, - "variant_effect_measurements": [], - } - equivalent_nt_variants[related_allele_id]["variant_effect_measurements"].append(variant) - - variants_by_allele_id[allele_id]["equivalent_nt"] = [ - equivalent_nt_variants[related_allele_id] for related_allele_id in equivalent_nt_variants - ] - - return [variants_by_allele_id[allele_id] for allele_id in request.clingen_allele_ids] - - @router.get( "/variants/{urn}", status_code=200, diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index ad02023df..c32abcab6 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -1,15 +1,12 @@ from datetime import date -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel -from mavedb.view_models.mapped_variant import MappedVariant, SavedMappedVariant - -if TYPE_CHECKING: - from mavedb.view_models.score_set import ShortScoreSet +from mavedb.view_models.mapped_variant import SavedMappedVariant class VariantEffectMeasurementBase(BaseModel): @@ -74,35 +71,3 @@ class VariantEffectMeasurement(SavedVariantEffectMeasurement): """Variant effect measurement view model returned to most clients""" pass - - -class VariantEffectMeasurementWithShortScoreSet(SavedVariantEffectMeasurement): - """Variant effect measurement view model with mapped variants and a limited set of score set details""" - - score_set: "ShortScoreSet" - mapped_variants: list[MappedVariant] - - -class ClingenAlleleIdVariantLookupsRequest(BaseModel): - """A request to search for variants matching a list of ClinGen allele IDs""" - - clingen_allele_ids: list[str] - - -class ClingenAlleleVariants(BaseModel): - """The assayed variants sharing one ClinGen allele id, with their variant effect measurements. - - An allele-side grouping (keyed by ClinGen allele id), it is the mapped-allele view of "which - measurements share this molecular identity," not the assayed variant itself.""" - - clingen_allele_id: str - variant_effect_measurements: list[VariantEffectMeasurementWithShortScoreSet] - - -class ClingenAlleleIdVariantLookupResponse(BaseModel): - """Response model for a variant lookup by ClinGen allele ID""" - - clingen_allele_id: str - exact_match: Optional[ClingenAlleleVariants] = None - equivalent_nt: list[ClingenAlleleVariants] = [] - equivalent_aa: list[ClingenAlleleVariants] = [] diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index 6c49f6713..e10cf95bd 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -223,6 +223,39 @@ def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set assert all(m.relationship == "nucleotide_encoding" for m in pa_page) +@pytest.mark.integration +def test_sibling_nt_pulled_onto_ca_page_with_flag(session, setup_lib_db_with_score_set): + """``include_nucleotide_siblings`` widens a CA page through the protein consequence: the sibling nt + change encoding the same protein is pulled in as a ``nucleotide_encoding`` while the direct measurement + is unchanged. A no-op for a PA query, which already returns every encoding.""" + score_set = setup_lib_db_with_score_set + nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") + nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + ra = _record(session, a, assay_level="cdna") + _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, prot) + + c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) + rc = _record(session, c, assay_level="cdna") + _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, prot) + + result = get_allele_measurements(session, "CA111", user_data=_user_data(session), include_nucleotide_siblings=True) + by_urn = {m.variant_urn: m for m in result} + assert set(by_urn) == {a.urn, c.urn} + assert by_urn[a.urn].relationship == "direct" + assert by_urn[c.urn].relationship == "nucleotide_encoding" + # Display order: the directly-measured change precedes its sibling nucleotide encoding. + assert [m.variant_urn for m in result] == [a.urn, c.urn] + + # No-op for a protein anchor — it already returns every nt encoding. + pa = get_allele_measurements(session, "PA9", user_data=_user_data(session), include_nucleotide_siblings=True) + assert {m.variant_urn for m in pa} == {a.urn, c.urn} + + @pytest.mark.integration def test_private_score_set_measurement_excluded(session, setup_lib_db_with_score_set): """Score-set READ gates inclusion: a measurement in a score set the caller cannot read never leaks, diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 7d75739db..e0236a134 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -104,6 +104,42 @@ def test_superseded_measurement_is_opt_in(client, session, data_provider, data_f assert body[0]["supersededByScoreSet"] == newer["urn"] +def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_files, setup_router_db): + """The sibling nt bucket is a discovery opt-in: a different DNA variant encoding the same protein + consequence is absent by default and pulled in as a ``nucleotide_encoding`` under + ``include_nucleotide_siblings``.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + # Two coding measurements encoding the same protein consequence (PA9), each carrying its own CA. + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + default = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert {m["variantUrn"] for m in default.json()} == {f"{score_set['urn']}#1"} + + widened = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + assert widened.status_code == 200 + by_urn = {m["variantUrn"]: m for m in widened.json()} + assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} + assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" + assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" + + def test_anonymous_cannot_see_private_measurement( client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides ): From 0a365d05f6f11714d8f9e4b8df8d2f19b9fbbbd2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 10 Jul 2026 13:45:11 -0700 Subject: [PATCH 69/93] refactor(annotations): rename AnnotationLayer to SequenceLevel and type assay_level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the AnnotationLayer enum to SequenceLevel to reflect that the same closed set (protein/cdna/genomic) serves several duties across the schema — the assayed level, the alignment level, and an allele's level — rather than reading as dcd-mapping's QC vocabulary alone. The name also harmonizes with the *_level columns and fields it types. Carry the enum through to consumers so assay_level is documented as a closed set instead of a free string: - type assay_level as Optional[SequenceLevel] on the lean-variant, variant-detail, and allele-measurement responses, so OpenAPI emits the enum members - type the matching lib transit dataclasses the same way, coercing the stored column string to the enum at each construction boundary - rename the mapping worker's local to sequence_level where it holds the enum (the dcd wire-code loop var keeps its name) The stored values and wire format are unchanged (SequenceLevel is a str enum), so no data migration is required and existing clients are unaffected. --- .../migrate_target_gene_mapping_qc.py | 40 +++++++++---------- src/mavedb/__init__.py | 2 +- src/mavedb/lib/allele_measurements.py | 14 +++---- src/mavedb/lib/cat_vrs.py | 10 ++--- src/mavedb/lib/score_set_variants.py | 36 ++++++++++------- src/mavedb/lib/variant_detail.py | 14 +++---- src/mavedb/models/enums/annotation_layer.py | 31 -------------- src/mavedb/models/enums/sequence_level.py | 34 ++++++++++++++++ src/mavedb/models/mapped_variant.py | 4 +- src/mavedb/models/mapping_record.py | 24 +++++++++-- src/mavedb/models/target_gene_mapping.py | 4 +- src/mavedb/view_models/allele_measurement.py | 3 +- src/mavedb/view_models/lean_variant.py | 5 ++- src/mavedb/view_models/mapped_variant.py | 4 +- src/mavedb/view_models/target_gene_mapping.py | 4 +- src/mavedb/view_models/variant_detail.py | 3 +- .../worker/jobs/variant_processing/mapping.py | 20 +++++----- .../variant_processing/reverse_translation.py | 12 +++--- tests/lib/test_score_set_variants.py | 2 +- .../jobs/variant_processing/test_mapping.py | 10 ++--- .../test_reverse_translation.py | 16 ++++---- 21 files changed, 160 insertions(+), 132 deletions(-) delete mode 100644 src/mavedb/models/enums/annotation_layer.py create mode 100644 src/mavedb/models/enums/sequence_level.py diff --git a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py index 4e793ffe5..41eec4665 100644 --- a/alembic/manual_migrations/migrate_target_gene_mapping_qc.py +++ b/alembic/manual_migrations/migrate_target_gene_mapping_qc.py @@ -76,7 +76,7 @@ from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import from mavedb.db.session import SessionLocal from mavedb.lib.variants import HGVS_G_REGEX, HGVS_P_REGEX, get_hgvs_from_post_mapped -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene @@ -102,18 +102,18 @@ _NUCLEOTIDE_ALPHABET = frozenset("ACGTUNRYSWKMBDHV-") -def _layer_from_hgvs(hgvs: str) -> Optional[AnnotationLayer]: - """Classify an HGVS string into an :class:`AnnotationLayer`.""" +def _layer_from_hgvs(hgvs: str) -> Optional[SequenceLevel]: + """Classify an HGVS string into an :class:`SequenceLevel`.""" if HGVS_G_REGEX.search(hgvs): - return AnnotationLayer.genomic + return SequenceLevel.genomic if HGVS_P_REGEX.search(hgvs): - return AnnotationLayer.protein + return SequenceLevel.protein if _HGVS_C_REGEX.search(hgvs): - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[AnnotationLayer]: +def _layer_from_vrs_sequence(post_mapped: dict) -> Optional[SequenceLevel]: """ Infer layer from VRS sequence alphabet (protein vs nucleotide). Returns protein if any allele/member sequence contains a non-nucleotide letter. @@ -150,15 +150,15 @@ def extract_sequences(obj): for seq in extract_sequences(post_mapped): found_seq = True if any(letter.upper() not in _NUCLEOTIDE_ALPHABET for letter in seq if letter.isalpha()): - return AnnotationLayer.protein + return SequenceLevel.protein if found_seq: - return AnnotationLayer.genomic + return SequenceLevel.genomic return None -def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: +def _alignment_level_for(target: TargetGene) -> Optional[SequenceLevel]: """Recover the alignment layer for legacy mapped variants of ``target``. Tries ``post_mapped_metadata`` first, then ``pre_mapped_metadata`` (same @@ -177,16 +177,16 @@ def _alignment_level_for(target: TargetGene) -> Optional[AnnotationLayer]: return None if has_genomic: - return AnnotationLayer.genomic + return SequenceLevel.genomic if has_protein: - return AnnotationLayer.protein + return SequenceLevel.protein if has_cdna: - return AnnotationLayer.cdna + return SequenceLevel.cdna return None -def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: +def _layer_from_sequence_type(target: TargetGene) -> Optional[SequenceLevel]: """Infer alignment layer from ``target_sequence.sequence_type`` and ``category``. Last-resort fallback for targets where all variant-level mappings failed @@ -201,16 +201,16 @@ def _layer_from_sequence_type(target: TargetGene) -> Optional[AnnotationLayer]: return None seq_type = target.target_sequence.sequence_type if seq_type == "protein": - return AnnotationLayer.protein + return SequenceLevel.protein if seq_type == "dna" and target.category is not None and target.category.value == "protein_coding": - return AnnotationLayer.genomic + return SequenceLevel.genomic return None def _populate_layer_by_target_id( db: Session, targets_by_score_set_id: dict[int, list[TargetGene]], - layer_by_target_id: dict[int, Optional[AnnotationLayer]], + layer_by_target_id: dict[int, Optional[SequenceLevel]], layer_via_hgvs_target_ids: set[int], layer_via_sequence_type_target_ids: set[int], ) -> None: @@ -344,7 +344,7 @@ def do_migration(db: Session) -> None: # Cache (target_gene_id, alignment_level, tool_version) -> persisted TargetGeneMapping # so we create one row per distinct combination and reuse it across every # mapped variant that maps into it. - cache: dict[tuple[int, AnnotationLayer, str], TargetGeneMapping] = {} + cache: dict[tuple[int, SequenceLevel, str], TargetGeneMapping] = {} total = db.scalar(sa.select(sa.func.count(MappedVariant.id))) or 0 print(f" {total} mapped variants to consider.") @@ -393,7 +393,7 @@ def do_migration(db: Session) -> None: # only need to determine it once per target -- and any single attributable mapped # variant for that target lets us attribute every sibling. The via_* sets track which # fallback was used so the diagnostic report shows attribution breadth. - layer_by_target_id: dict[int, Optional[AnnotationLayer]] = {} + layer_by_target_id: dict[int, Optional[SequenceLevel]] = {} layer_via_hgvs_target_ids: set[int] = set() layer_via_sequence_type_target_ids: set[int] = set() _populate_layer_by_target_id( @@ -420,7 +420,7 @@ def do_migration(db: Session) -> None: # Group ids to update by (tgm_id, level) so we can issue one bulk UPDATE # per group instead of a per-row ORM flush. - updates_by_group: dict[tuple[int, AnnotationLayer], list[int]] = defaultdict(list) + updates_by_group: dict[tuple[int, SequenceLevel], list[int]] = defaultdict(list) for row in chunk: last_id = row.id diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index 4669471c8..0b82b3395 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.6" +__version__ = "2026.2.6-dev" logger.info(f"MaveDB {__version__}") diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index a3048cb83..eaa988baf 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -37,7 +37,7 @@ from mavedb.lib.types.authentication import UserData from mavedb.lib.variants import variant_score from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_calibration import ScoreCalibration @@ -76,7 +76,7 @@ class AlleleMeasurement: variant_urn: str score: Optional[float] - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] relationship: str assay_level_hgvs: Optional[str] submitted_hgvs: Optional[str] @@ -169,7 +169,7 @@ def get_allele_measurements( return [] anchor_ids = [row.id for row in anchor] - entry_is_protein = any(row.level == AnnotationLayer.protein.value for row in anchor) + entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record @@ -184,7 +184,7 @@ def get_allele_measurements( .where(anchor_link.allele_id.in_(anchor_ids)) .where(anchor_link.live_at(as_of)) .where(MappingRecordAllele.live_at(as_of)) - .where(Allele.level == AnnotationLayer.protein.value) + .where(Allele.level == SequenceLevel.protein.value) .distinct() ).all() @@ -240,19 +240,19 @@ def get_allele_measurements( # branch is unreachable (a record links the anchor only via itself or its protein consequence). if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct - elif measured_allele.level == AnnotationLayer.protein.value: + elif measured_allele.level == SequenceLevel.protein.value: relationship = MeasurementRelationship.protein_consequence else: relationship = MeasurementRelationship.nucleotide_encoding - assay_level = record.assay_level + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None measurement = AlleleMeasurement( variant_urn=variant.urn or "", score=variant_score(variant), assay_level=assay_level, relationship=relationship, assay_level_hgvs=record.hgvs_assay_level, - submitted_hgvs=variant.hgvs_pro if assay_level == AnnotationLayer.protein.value else variant.hgvs_nt, + submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", primary_classification=_preferred_classification(db, variant, user_data=user_data), diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 498c13542..10f1920ec 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -31,7 +31,7 @@ from mavedb.lib.annotation.util import vrs_object_from_mapped_variant from mavedb.lib.logging.context import logging_context from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) @@ -60,7 +60,7 @@ class CatVrsMode(str, Enum): REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. -_NUCLEOTIDE_LEVELS = {AnnotationLayer.genomic.value, AnnotationLayer.cdna.value} +_NUCLEOTIDE_LEVELS = {SequenceLevel.genomic.value, SequenceLevel.cdna.value} @dataclass @@ -80,11 +80,11 @@ def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> the (defining, member) level pair, never on a chain between members. """ # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. - if defining_level == AnnotationLayer.protein.value: + if defining_level == SequenceLevel.protein.value: return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None # Mode 1 — nt measured (genomic or cdna). - if member_level == AnnotationLayer.protein.value: + if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF if member_level in _NUCLEOTIDE_LEVELS: return CatVrsRelation.COORDINATE_REPRESENTATION_OF @@ -141,7 +141,7 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> return None defining_level = defining_allele.level - mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == AnnotationLayer.protein.value else CatVrsMode.PROJECTION + mode = CatVrsMode.REVERSE_TRANSLATION if defining_level == SequenceLevel.protein.value else CatVrsMode.PROJECTION members: list[VrsAllele | CisPhasedBlock | iriReference] = [defining_vrs] member_relations: dict[str, CatVrsRelation] = {} diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index b2a2e060a..f0c75390f 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -37,7 +37,7 @@ from mavedb.lib.hgvs import parse_simple_substitution from mavedb.lib.variants import score_from_variant_data from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet @@ -84,7 +84,7 @@ class LeanVariantRecord: """One pre-chewed per-variant record: the selection key (``variant_urn``), the baseline ``score``, a representative (lossy) VEP ``consequence``, the bridge identifiers into the annotation dimensions (``clingen_allele_id``, ``assay_level_digest``), the submitted HGVS at each level, and the mapped - (reference-frame) representation as an ``assay_level`` pointer (an ``AnnotationLayer`` value naming + (reference-frame) representation as an ``assay_level`` pointer (an ``SequenceLevel`` value naming the measured/canonical slot) plus a ``mapped`` :class:`MappedTriple`. ``mapped[assay_level]`` is the measured representation; ``mapped.cdna`` is the level-invariant search key. Any field is ``None`` when its source is absent (unmapped variant → null ``assay_level`` + empty triple).""" @@ -97,10 +97,16 @@ class LeanVariantRecord: hgvs_nt: Optional[HgvsField] hgvs_pro: Optional[HgvsField] hgvs_splice: Optional[HgvsField] - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] mapped: MappedTriple +def _level(value: Optional[str]) -> Optional[SequenceLevel]: + """Coerce a stored assay-level string (the ``mapping_records.assay_level`` column) into the closed + :class:`SequenceLevel` set. ``None`` for an absent value.""" + return SequenceLevel(value) if value else None + + def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: """Wrap an HGVS string as an :class:`HgvsField`, attaching a parsed block when it is a placeable simple substitution. ``None`` for an absent string.""" @@ -114,7 +120,7 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: def _mapped_triple( *, - assay_level: Optional[str], + assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], sibling_level: Optional[str], sibling_hgvs: Optional[str], @@ -133,19 +139,19 @@ def _mapped_triple( """ slots: dict[str, Optional[HgvsField]] = {} # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. - if assay_level == AnnotationLayer.protein.value: - slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(assay_level_hgvs) - elif assay_level in (AnnotationLayer.cdna.value, AnnotationLayer.genomic.value): - slots[assay_level] = _hgvs_field_for_str(assay_level_hgvs) + if assay_level == SequenceLevel.protein: + slots[SequenceLevel.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): + slots[assay_level.value] = _hgvs_field_for_str(assay_level_hgvs) # The other nucleotide level, from the projection_group sibling (null where unpopulated). if sibling_level is not None: slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) - slots[AnnotationLayer.protein.value] = _hgvs_field_for_str(protein_hgvs) + slots[SequenceLevel.protein.value] = _hgvs_field_for_str(protein_hgvs) return MappedTriple( - genomic=slots.get(AnnotationLayer.genomic.value), - cdna=slots.get(AnnotationLayer.cdna.value), - protein=slots.get(AnnotationLayer.protein.value), + genomic=slots.get(SequenceLevel.genomic.value), + cdna=slots.get(SequenceLevel.cdna.value), + protein=slots.get(SequenceLevel.protein.value), ) @@ -258,7 +264,7 @@ def get_lean_score_set_variants( # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. .where(prot_link.live_at(as_of)) # Only the protein-level allele. - .where(prot_allele.level == AnnotationLayer.protein.value) + .where(prot_allele.level == SequenceLevel.protein.value) # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives # per record — inert today (only one protein allele exists). .order_by(prot_link.mapping_record_id, prot_allele.id) @@ -279,9 +285,9 @@ def get_lean_score_set_variants( hgvs_nt=_hgvs_field_for_str(row.hgvs_nt), hgvs_pro=_hgvs_field_for_str(row.hgvs_pro), hgvs_splice=_hgvs_field_for_str(row.hgvs_splice), - assay_level=row.assay_level, + assay_level=_level(row.assay_level), mapped=_mapped_triple( - assay_level=row.assay_level, + assay_level=_level(row.assay_level), assay_level_hgvs=row.hgvs_assay_level, sibling_level=row.sibling_level, sibling_hgvs=row.sibling_hgvs, diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index e833545a8..1f3cde103 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -28,7 +28,7 @@ from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant from mavedb.lib.score_calibrations import calibration_preference_key -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -114,7 +114,7 @@ class VariantDetail: classifications: list[VariantClassificationRecord] # Flat, UI-ergonomic assay-level fields. - assay_level: Optional[str] + assay_level: Optional[SequenceLevel] target_hgvs: Optional[str] # submitted, target/assay coordinates reference_hgvs: Optional[str] # mapped, reference coordinates, assay level assay_level_digest: Optional[str] @@ -170,7 +170,7 @@ def _classifications_for_variant( ] -def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> AlleleDerivation: +def _derivation_for(*, is_authoritative: bool, assay_level: Optional[SequenceLevel]) -> AlleleDerivation: """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* @@ -188,16 +188,16 @@ def _derivation_for(*, is_authoritative: bool, assay_level: Optional[str]) -> Al """ if is_authoritative: return AlleleDerivation.AUTHORITATIVE - if assay_level == AnnotationLayer.protein.value: + if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE return AlleleDerivation.PROJECTION -def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[str]) -> Optional[str]: +def _submitted_assay_level_hgvs(variant: Variant, assay_level: Optional[SequenceLevel]) -> Optional[str]: """The depositor-submitted HGVS in the variant's assay frame: protein for a protein assay, otherwise the nucleotide expression (genomic or coding share ``hgvs_nt`` and ``hgvs_splice`` is never an index column).""" - if assay_level == AnnotationLayer.protein.value: + if assay_level == SequenceLevel.protein.value: return variant.hgvs_pro return variant.hgvs_nt @@ -226,7 +226,7 @@ def get_variant_detail( record = db.scalar( select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) ) - assay_level = record.assay_level if record is not None else None + assay_level = SequenceLevel(record.assay_level) if record is not None else None reference_hgvs = record.hgvs_assay_level if record is not None else None # The live allele links: the authoritative allele gives the assay-level digest + ClinGen id; all diff --git a/src/mavedb/models/enums/annotation_layer.py b/src/mavedb/models/enums/annotation_layer.py deleted file mode 100644 index 2115f8e99..000000000 --- a/src/mavedb/models/enums/annotation_layer.py +++ /dev/null @@ -1,31 +0,0 @@ -from enum import Enum - - -class AnnotationLayer(str, Enum): - """Annotation layer for a variant mapping result. - - Mirrors the ``AnnotationLayer`` enum produced by the dcd-mapping QC API. - Values use full names so they round-trip readably through the database - column; the dcd-mapping payload uses short single-character codes - (``p`` / ``c`` / ``g``) which the worker translates via :func:`from_wire`. - """ - - protein = "protein" - cdna = "cdna" - genomic = "genomic" - - @classmethod - def from_wire(cls, code: str) -> "AnnotationLayer": - """Translate a dcd-mapping single-character code into an ``AnnotationLayer``.""" - try: - return _WIRE_TO_LAYER[code] - except KeyError as exc: - raise ValueError(f"Unknown annotation_level wire code: {code!r}") from exc - - -# Module-level so it doesn't get mistaken for an enum member. -_WIRE_TO_LAYER: dict[str, AnnotationLayer] = { - "p": AnnotationLayer.protein, - "c": AnnotationLayer.cdna, - "g": AnnotationLayer.genomic, -} diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py new file mode 100644 index 000000000..016c36f96 --- /dev/null +++ b/src/mavedb/models/enums/sequence_level.py @@ -0,0 +1,34 @@ +from enum import Enum + + +class SequenceLevel(str, Enum): + """The molecular sequence level of a variant representation: genomic DNA, coding DNA, or protein. + + A single, duty-neutral closed set reused across several columns that each carry a different + semantic meaning: the level a variant was *assayed* at (``assay_level``), the level dcd-mapping + *aligned* it at (``alignment_level``), and the level of a stored allele (``level``). + + Values use full names so they round-trip readably through the database column; the dcd-mapping + payload uses short single-character codes (``p`` / ``c`` / ``g``) which the worker translates via + :func:`from_wire`. + """ + + protein = "protein" + cdna = "cdna" + genomic = "genomic" + + @classmethod + def from_wire(cls, code: str) -> "SequenceLevel": + """Translate a dcd-mapping single-character code into a ``SequenceLevel``.""" + try: + return _WIRE_TO_LEVEL[code] + except KeyError as exc: + raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + + +# Module-level so it doesn't get mistaken for an enum member. +_WIRE_TO_LEVEL: dict[str, SequenceLevel] = { + "p": SequenceLevel.protein, + "c": SequenceLevel.cdna, + "g": SequenceLevel.genomic, +} diff --git a/src/mavedb/models/mapped_variant.py b/src/mavedb/models/mapped_variant.py index 40325cf17..10914f778 100644 --- a/src/mavedb/models/mapped_variant.py +++ b/src/mavedb/models/mapped_variant.py @@ -7,7 +7,7 @@ from mavedb.db.base import Base from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_variant_mapped_variant import gnomad_variants_mapped_variants_association_table if TYPE_CHECKING: @@ -44,7 +44,7 @@ class MappedVariant(Base): # Per-mapping QC annotations from dcd-mapping ScoreAnnotation. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) diff --git a/src/mavedb/models/mapping_record.py b/src/mavedb/models/mapping_record.py index 132a69b8b..dda9ef16e 100644 --- a/src/mavedb/models/mapping_record.py +++ b/src/mavedb/models/mapping_record.py @@ -9,7 +9,7 @@ from mavedb.db.base import Base from mavedb.db.mixins import ValidTime from mavedb.lib.hgvs import extract_accession -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from .mapping_record_allele import MappingRecordAllele @@ -30,8 +30,24 @@ class MappingRecord(ValidTime, Base): vrs_digest = Column(String, nullable=True) pre_mapped: Optional[Any] = Column(JSONB(none_as_null=True), nullable=True) - # Level at which the variant was *assayed* — distinct from alignment_level (QC). - assay_level = Column(String(length=16), nullable=False) + # Level at which the variant was *assayed* — distinct from alignment_level (QC). Same closed + # SequenceLevel set as its sibling below. The prod CHECK (ck_mapping_records_assay_level_valid) + # predates this column being typed as an Enum; native_enum=False keeps it a VARCHAR(16), so + # declaring the Enum here needs no migration — it self-documents the model and gives the + # metadata-built (test) schema the same guard. + assay_level: Mapped[SequenceLevel] = Column( + # Distinct `name` so the generated CHECK doesn't collide with alignment_level's in this same + # table (both default to "sequencelevel", which Postgres rejects as a duplicate per table). + Enum( + SequenceLevel, + name="assay_level", + create_constraint=True, + length=16, + native_enum=False, + validate_strings=True, + ), + nullable=False, + ) hgvs_assay_level = Column(String, nullable=True) @hybrid_property @@ -60,7 +76,7 @@ def _transcript_expression(cls): # Per-mapping QC fields from dcd-mapping. alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=True, ) at_mismatched_locus = Column(Boolean, nullable=True) diff --git a/src/mavedb/models/target_gene_mapping.py b/src/mavedb/models/target_gene_mapping.py index 52761e34b..5ead4a887 100644 --- a/src/mavedb/models/target_gene_mapping.py +++ b/src/mavedb/models/target_gene_mapping.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import Mapped, relationship from mavedb.db.base import Base -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel if TYPE_CHECKING: from mavedb.models.mapped_variant import MappedVariant @@ -36,7 +36,7 @@ class TargetGeneMapping(Base): target_gene: Mapped["TargetGene"] = relationship("TargetGene", back_populates="target_gene_mappings") alignment_level = Column( - Enum(AnnotationLayer, create_constraint=True, length=16, native_enum=False, validate_strings=True), + Enum(SequenceLevel, create_constraint=True, length=16, native_enum=False, validate_strings=True), nullable=False, ) preferred = Column(Boolean, nullable=False, default=False, server_default="false") diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index d61197524..8ac03275f 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -8,6 +8,7 @@ from typing import Optional from mavedb.lib.allele_measurements import MeasurementRelationship +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -27,7 +28,7 @@ class AlleleMeasurement(BaseModel): variant_urn: str score: Optional[float] = None - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None relationship: MeasurementRelationship assay_level_hgvs: Optional[str] = None submitted_hgvs: Optional[str] = None diff --git a/src/mavedb/view_models/lean_variant.py b/src/mavedb/view_models/lean_variant.py index 31f163524..474484e40 100644 --- a/src/mavedb/view_models/lean_variant.py +++ b/src/mavedb/view_models/lean_variant.py @@ -10,6 +10,7 @@ from typing import Optional +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.base.base import BaseModel @@ -52,7 +53,7 @@ class LeanVariant(BaseModel): ``variantUrn`` is the universal selection key; ``assayLevelDigest`` bridges into the digest-keyed annotation dimensions. The submitted HGVS (``hgvsNt``/``hgvsPro``/``hgvsSplice``, target frame) carry the depositor's frame for the heatmap's raw↔mapped toggle. The mapped (reference) frame is the - ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``AnnotationLayer`` value) naming + ``mapped`` :class:`MappedTriple` plus the ``assayLevel`` pointer (an ``SequenceLevel`` value) naming the measured/canonical slot: ``mapped[assayLevel]`` is the measured representation and ``mapped.cdna`` the level-invariant search key. Fields are omitted when null. """ @@ -65,7 +66,7 @@ class LeanVariant(BaseModel): hgvs_nt: Optional[HgvsField] = None hgvs_pro: Optional[HgvsField] = None hgvs_splice: Optional[HgvsField] = None - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None mapped: MappedTriple = MappedTriple() class Config: diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 2d4856c88..218f0ba8d 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -7,7 +7,7 @@ from pydantic import model_validator from mavedb.lib.validation.exceptions import ValidationError -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel from mavedb.view_models.target_gene_mapping import SavedTargetGeneMapping, TargetGeneMapping @@ -28,7 +28,7 @@ class MappedVariantBase(BaseModel): current: bool # Per-mapping QC annotations/warnings. - alignment_level: Optional[AnnotationLayer] = None + alignment_level: Optional[SequenceLevel] = None at_mismatched_locus: Optional[bool] = None near_gap: Optional[bool] = None diff --git a/src/mavedb/view_models/target_gene_mapping.py b/src/mavedb/view_models/target_gene_mapping.py index 13e64d1a7..535508a42 100644 --- a/src/mavedb/view_models/target_gene_mapping.py +++ b/src/mavedb/view_models/target_gene_mapping.py @@ -7,13 +7,13 @@ from datetime import date from typing import Any, Optional -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models import record_type_validator, set_record_type from mavedb.view_models.base.base import BaseModel class TargetGeneMappingBase(BaseModel): - alignment_level: AnnotationLayer + alignment_level: SequenceLevel preferred: bool = False reference_assembly: Optional[str] = None diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 1ae03def8..b14d2fa7a 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -8,6 +8,7 @@ from typing import Any, Optional +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.allele_annotation import AlleleAnnotations from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -76,7 +77,7 @@ class VariantDetail(BaseModel): counts: Optional[dict[str, Any]] = None classifications: list[VariantClassification] = [] - assay_level: Optional[str] = None + assay_level: Optional[SequenceLevel] = None target_hgvs: Optional[str] = None reference_hgvs: Optional[str] = None assay_level_digest: Optional[str] = None diff --git a/src/mavedb/worker/jobs/variant_processing/mapping.py b/src/mavedb/worker/jobs/variant_processing/mapping.py index b2f588b72..14d62fcdf 100644 --- a/src/mavedb/worker/jobs/variant_processing/mapping.py +++ b/src/mavedb/worker/jobs/variant_processing/mapping.py @@ -29,7 +29,7 @@ from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.job_pipeline import FailureCategory @@ -184,7 +184,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan for annotation_layer in reference_metadata[target_gene_identifier]["layers"]: # ``annotation_layer`` arrives as a dcd-mapping wire code (``p``/``c``/``g``); # we persist metadata under the corresponding full-name enum value. - layer_name = AnnotationLayer.from_wire(annotation_layer).value + layer_name = SequenceLevel.from_wire(annotation_layer).value layer_premapped = reference_metadata[target_gene_identifier]["layers"][annotation_layer].get( "computed_reference_sequence" ) @@ -225,7 +225,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan target_gene_mapping = TargetGeneMapping( target_gene=target_gene, - alignment_level=AnnotationLayer.from_wire(level_value), + alignment_level=SequenceLevel.from_wire(level_value), preferred=bool(tm.get("preferred", False)), reference_assembly=tm.get("reference_assembly"), reference_accession=tm.get("reference_accession"), @@ -314,7 +314,7 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan pre_mapped_allele: dict = mapped_score.get("pre_mapped") or {} post_mapped_allele: dict = mapped_score.get("post_mapped") or {} - annotation_layer = AnnotationLayer.from_wire(score_alignment_level) + sequence_level = SequenceLevel.from_wire(score_alignment_level) assay_level_hgvs = get_hgvs_from_post_mapped(post_mapped_allele, combine_cis=True) # dcd-mapping guarantees every mapped score is attributable to a TargetGeneMapping @@ -331,13 +331,13 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan variant_id=variant.id, vrs_digest=pre_mapped_allele.get("id"), pre_mapped=pre_mapped_allele or None, - assay_level=annotation_layer, + assay_level=sequence_level, hgvs_assay_level=assay_level_hgvs, mapped_date=mapping_results["mapped_date"], vrs_version=mapped_score.get("vrs_version", None), mapping_api_version=tool_version, target_gene_mapping_id=target_gene_mapping_row.id, - alignment_level=annotation_layer, + alignment_level=sequence_level, at_mismatched_locus=mapped_score.get("at_mismatched_locus"), near_gap=mapped_score.get("near_gap"), ) @@ -380,10 +380,10 @@ async def map_variants_for_score_set(ctx: dict, job_id: int, job_manager: JobMan if post_mapped_allele: allele_draft = AlleleDbModel( vrs_digest=post_mapped_allele["id"], - level=annotation_layer, - hgvs_g=assay_level_hgvs if annotation_layer == AnnotationLayer.genomic else None, - hgvs_c=assay_level_hgvs if annotation_layer == AnnotationLayer.cdna else None, - hgvs_p=assay_level_hgvs if annotation_layer == AnnotationLayer.protein else None, + level=sequence_level, + hgvs_g=assay_level_hgvs if sequence_level == SequenceLevel.genomic else None, + hgvs_c=assay_level_hgvs if sequence_level == SequenceLevel.cdna else None, + hgvs_p=assay_level_hgvs if sequence_level == SequenceLevel.protein else None, post_mapped=post_mapped_allele, ) authoritative_allele = get_or_create_allele(job_manager.db, allele_draft) diff --git a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py index 8d861270a..a6cdfc257 100644 --- a/src/mavedb/worker/jobs/variant_processing/reverse_translation.py +++ b/src/mavedb/worker/jobs/variant_processing/reverse_translation.py @@ -37,7 +37,7 @@ from mavedb.lib.variant_translations import get_or_create_allele from mavedb.lib.vrs_utils import translate_hgvs_to_variation from mavedb.models.allele import Allele as AlleleDbModel -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason @@ -221,7 +221,7 @@ async def reverse_translate_variants_for_score_set( ) .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) .where(TargetGene.score_set_id == score_set_id) - .where(TargetGeneMapping.alignment_level == AnnotationLayer.cdna) + .where(TargetGeneMapping.alignment_level == SequenceLevel.cdna) .where(TargetGeneMapping.reference_accession.isnot(None)) .order_by(TargetGeneMapping.id) ) @@ -398,18 +398,18 @@ async def reverse_translate_variants_for_score_set( # failed, yielding a well-formed one-member (coding only) group rather than a desync. The # group id is assigned once per pair here, so the two members are guaranteed to carry the # same id even though they translate and link independently below. - members: list[tuple[str, AnnotationLayer, str, int | None]] = [] + members: list[tuple[str, SequenceLevel, str, int | None]] = [] for group_id, pair in enumerate(result.projection_pairs): - members.append((pair.hgvs_c, AnnotationLayer.cdna, "hgvs_c", group_id)) + members.append((pair.hgvs_c, SequenceLevel.cdna, "hgvs_c", group_id)) if pair.hgvs_g is not None: - members.append((pair.hgvs_g, AnnotationLayer.genomic, "hgvs_g", group_id)) + members.append((pair.hgvs_g, SequenceLevel.genomic, "hgvs_g", group_id)) # The protein consequence is the apex of the equivalence set — shared across every # pair, a member of none — so it carries no group (None). Prediction parens # (p.(Ala222Val)) are stripped before translation and storage. None for protein-assay # inputs, where the protein is already the authoritative allele. if result.hgvs_p: - members.append((strip_protein_prediction_parens(result.hgvs_p), AnnotationLayer.protein, "hgvs_p", None)) + members.append((strip_protein_prediction_parens(result.hgvs_p), SequenceLevel.protein, "hgvs_p", None)) for hgvs, level, hgvs_field, projection_group in members: # A candidate may not be translatable (intronic projection, malformed expression). diff --git a/tests/lib/test_score_set_variants.py b/tests/lib/test_score_set_variants.py index 4c0cbb5b0..6ac84bb12 100644 --- a/tests/lib/test_score_set_variants.py +++ b/tests/lib/test_score_set_variants.py @@ -275,7 +275,7 @@ def test_ordered_by_variant_number(session, setup_lib_db_with_score_set): @pytest.mark.integration -def test_as_of_reconstructs_the_historical_annotation_layer(session, setup_lib_db_with_score_set): +def test_as_of_reconstructs_the_historical_assay_level(session, setup_lib_db_with_score_set): """A re-map retires the old record and inserts a new one; as_of reconstructs the mapped layer live at a past instant, while the immutable submitted HGVS and score are unaffected.""" score_set = setup_lib_db_with_score_set diff --git a/tests/worker/jobs/variant_processing/test_mapping.py b/tests/worker/jobs/variant_processing/test_mapping.py index 96321d816..d4134ab05 100644 --- a/tests/worker/jobs/variant_processing/test_mapping.py +++ b/tests/worker/jobs/variant_processing/test_mapping.py @@ -13,7 +13,7 @@ from mavedb.lib.mapping import EXCLUDED_PREMAPPED_ANNOTATION_KEYS from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.enums.mapping_state import MappingState from mavedb.models.mapping_record import MappingRecord @@ -514,7 +514,7 @@ async def dummy_mapping_job(): assert result.status == JobStatus.SUCCEEDED cdna_tgms = ( - session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == AnnotationLayer.cdna).all() + session.query(TargetGeneMapping).filter(TargetGeneMapping.alignment_level == SequenceLevel.cdna).all() ) assert len(cdna_tgms) == len(sample_score_set.target_genes) tgm = cdna_tgms[0] @@ -894,7 +894,7 @@ async def dummy_mapping_job(): session.commit() mapping_record = MappingRecord( variant_id=variant.id, - assay_level=AnnotationLayer.genomic, + assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) @@ -902,7 +902,7 @@ async def dummy_mapping_job(): session.commit() # Link the prior record to an authoritative allele. Re-mapping must retire this link too, # not just the record — a live link dangling off a retired record breaks the temporal model. - prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=AnnotationLayer.genomic.value) + prior_allele = Allele(vrs_digest="ga4gh:VA.prior", level=SequenceLevel.genomic.value) session.add(prior_allele) session.commit() prior_link = MappingRecordAllele( @@ -1421,7 +1421,7 @@ async def test_map_variants_for_score_set_updates_current_mapped_variants( for variant in variants: mapping_record = MappingRecord( variant_id=variant.id, - assay_level=AnnotationLayer.genomic, + assay_level=SequenceLevel.genomic, mapped_date=date(2023, 1, 1), mapping_api_version="v1.0.0", ) diff --git a/tests/worker/jobs/variant_processing/test_reverse_translation.py b/tests/worker/jobs/variant_processing/test_reverse_translation.py index 7bc33cf99..030d17020 100644 --- a/tests/worker/jobs/variant_processing/test_reverse_translation.py +++ b/tests/worker/jobs/variant_processing/test_reverse_translation.py @@ -18,7 +18,7 @@ from mavedb.lib.types.workflow import JobExecutionOutcome from mavedb.models.allele import Allele -from mavedb.models.enums.annotation_layer import AnnotationLayer +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.enums.job_pipeline import JobStatus from mavedb.models.enums.target_category import TargetCategory from mavedb.models.job_run import JobRun @@ -315,14 +315,14 @@ async def test_creates_genomic_and_coding_candidate_alleles( assert len(non_auth_links) == 2 genomic_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.genomic").one() - assert genomic_allele.level == AnnotationLayer.genomic.value + assert genomic_allele.level == SequenceLevel.genomic.value assert genomic_allele.hgvs_g == g_candidate assert genomic_allele.hgvs_c is None assert genomic_allele.transcript == "NC_000001.11" assert genomic_allele.post_mapped == {"type": "Allele", "id": "ga4gh:VA.genomic"} coding_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.coding").one() - assert coding_allele.level == AnnotationLayer.cdna.value + assert coding_allele.level == SequenceLevel.cdna.value assert coding_allele.hgvs_c == c_candidate assert coding_allele.hgvs_g is None assert coding_allele.transcript == "NM_000001.1" @@ -431,7 +431,7 @@ async def test_protein_consequence_is_persisted_as_a_protein_allele( await _reverse_translate(session, mock_worker_ctx, sample_independent_reverse_translation_run) protein_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:VA.protein").one() - assert protein_allele.level == AnnotationLayer.protein.value + assert protein_allele.level == SequenceLevel.protein.value assert protein_allele.hgvs_p == p_stripped # stored without the prediction parens # Linked to the record as a non-authoritative member of the equivalence set, but grouped @@ -485,7 +485,7 @@ async def test_cis_phased_multivariant_candidate_is_persisted_as_a_block( assert result.data == {"translated": 1, "failed": 0, "skipped": 0, "alleles_created": 2} block_allele = session.query(Allele).filter(Allele.vrs_digest == "ga4gh:CPB.block").one() - assert block_allele.level == AnnotationLayer.genomic.value + assert block_allele.level == SequenceLevel.genomic.value # The full bracketed expression is stored verbatim; its accession anchors the row. assert block_allele.hgvs_g == block_candidate assert block_allele.transcript == "NC_000001.11" @@ -1152,7 +1152,7 @@ def _run_mapped_date(self, session, target_gene_id): session.query(TargetGeneMapping) .filter( TargetGeneMapping.target_gene_id == target_gene_id, - TargetGeneMapping.alignment_level == AnnotationLayer.genomic, + TargetGeneMapping.alignment_level == SequenceLevel.genomic, ) .first() .mapped_date @@ -1192,7 +1192,7 @@ async def test_uses_latest_cdna_row_within_the_run( session.add( TargetGeneMapping( target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.cdna, + alignment_level=SequenceLevel.cdna, reference_accession="NM_111111.1", preferred=False, tool_version="pytest.0.0", @@ -1253,7 +1253,7 @@ async def test_ignores_stale_cdna_row_from_a_different_run( session.add( TargetGeneMapping( target_gene_id=target_gene.id, - alignment_level=AnnotationLayer.cdna, + alignment_level=SequenceLevel.cdna, reference_accession="NM_888888.1", preferred=False, tool_version="pytest.0.0", From ccf5b69234997045a19a60af6f7306c1866b2787 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 10 Jul 2026 14:07:48 -0700 Subject: [PATCH 70/93] fix(docs): remove design references from endpoint documentation --- src/mavedb/lib/allele_annotations.py | 4 ++-- src/mavedb/lib/variant_detail.py | 2 +- src/mavedb/view_models/allele_measurement.py | 2 +- src/mavedb/view_models/variant_detail.py | 6 +++--- tests/routers/test_clingen_allele.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mavedb/lib/allele_annotations.py b/src/mavedb/lib/allele_annotations.py index 1b6cf112b..4e161dbf5 100644 --- a/src/mavedb/lib/allele_annotations.py +++ b/src/mavedb/lib/allele_annotations.py @@ -2,8 +2,8 @@ Gathers a set of alleles' external annotations — VEP consequence, gnomAD frequency, ClinVar assertions — keyed by ``vrs_digest``. The digest is the join key the serving envelopes use to ride -annotations *alongside* the spec-pure Cat-VRS members (design §8): Cat-VRS itself carries no -annotation slot, so annotations travel beside it in a flat digest-keyed map. +annotations *alongside* the spec-pure Cat-VRS members: Cat-VRS itself carries no annotation slot, so +annotations travel beside it in a flat digest-keyed map. All three sources are ``ValidTime`` and ``allele_id``-keyed with no reverse collection on ``Allele`` (they are navigated set-wise from the link tables). ``as_of`` reconstructs them at a past diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 1f3cde103..987e0e3ea 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -7,7 +7,7 @@ digest-keyed external-annotation map (:mod:`lib.allele_annotations`). Also carries the per-calibration functional ``classifications`` the variant falls into, and its version standing (``is_current`` / ``superseded_by_score_set``) so a superseded variant self-describes rather -than reading as current (design §9.2). +than reading as current. Temporal scope of ``as_of``: only the **molecular** layer is versioned — Cat-VRS membership and the VEP/gnomAD/ClinVar annotations reconstruct at the past instant. Scores are immutable (``Variant`` is diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 8ac03275f..592285d12 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -1,5 +1,5 @@ """Response view model for the variant page's measurements list -(``GET /clingen-alleles/{caid}/measurements``, design §7.5). +(``GET /clingen-alleles/{caid}/measurements``). The pydantic serialization boundary over the ``lib.allele_measurements`` transit dataclass (``from_attributes`` coerces it directly); aliases camelize per the shared base config. diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index b14d2fa7a..190f41d7a 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -1,4 +1,4 @@ -"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``, design §7.1). +"""Response view models for the assayed variant-detail view (``GET /variants/{urn}``). The pydantic serialization boundary over the ``lib.variant_detail`` transit dataclasses (``from_attributes`` coerces them directly); aliases camelize per the shared base config. Kept in its @@ -34,7 +34,7 @@ class AlleleIdentity(BaseModel): """The MaveDB molecular-identity facts for one of the variant's linked alleles. An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame - HGVS) are what the UI labels the per-level annotation panel by — never the digest (design §7.5). + HGVS) are what the UI labels the per-level annotation panel by — never the digest. Three independent axes: - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured @@ -58,7 +58,7 @@ class Config: class VariantDetail(BaseModel): - """The assayed variant-detail envelope (``GET /variants/{urn}``, design §7.1). + """The assayed variant-detail envelope (``GET /variants/{urn}``). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index e0236a134..97d4fed43 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 """Router tests for the ClinGen-allele measurements endpoint -(``GET /clingen-alleles/{caid}/measurements``, design §7.5). +(``GET /clingen-alleles/{caid}/measurements``). Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded From 19072d607b161545690cb960c7fa2662e3cd4352 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sun, 12 Jul 2026 13:43:33 -0700 Subject: [PATCH 71/93] feat(allele-measurements): rank current measurements before superseded Add an is_current dimension as the top precedence in _ordering_key so current measurements sort ahead of superseded ones, above the existing direct-vs-related, evidence-strength, published-date, and urn keys. - Extend _ordering_key with the current/superseded ordinal - Add pure unit tests pinning the full precedence chain and the null-published-date fallback - Add router tests covering direct-before-related, current-before- superseded, newest-published-first, and the urn tiebreak in the serialized response body --- src/mavedb/lib/allele_measurements.py | 10 ++- tests/lib/test_allele_measurements.py | 71 ++++++++++++++- tests/routers/test_clingen_allele.py | 119 +++++++++++++++++++++++++- 3 files changed, 191 insertions(+), 9 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index eaa988baf..9419ea244 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -131,13 +131,15 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: """Sort order: - 1. Direct measurements, then related (protein_consequence / nucleotide_encoding) - 2. Within each, the strongest evidence first (pathogenic wins ties) - 3. Within each, the newest-published first - 4. Within each, the URN for a stable tiebreak + 1. Current measurements first, then superseded (the latter only if ``include_superseded``) + 2. Direct measurements first, then related (protein_consequence / nucleotide_encoding) + 3. Within each, the strongest evidence first (pathogenic wins ties) + 4. Within each, the newest-published first + 5. Within each, the URN for a stable tiebreak """ magnitude, direction = classification_evidence_strength(measurement.primary_classification) return ( + 0 if measurement.is_current else 1, 0 if measurement.relationship == MeasurementRelationship.direct else 1, 0 if measurement.primary_classification is not None else 1, -magnitude, diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index e10cf95bd..d64b33e6a 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -15,7 +15,14 @@ pytest.importorskip("psycopg2") -from mavedb.lib.allele_measurements import get_allele_measurements +from datetime import date + +from mavedb.lib.allele_measurements import ( + AlleleMeasurement, + MeasurementRelationship, + _ordering_key, + get_allele_measurements, +) from mavedb.lib.types.authentication import UserData from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord @@ -402,3 +409,65 @@ def test_ordering_direct_first_then_strongest_pathogenic(session, setup_lib_db_w @pytest.mark.integration def test_unknown_clingen_id_returns_empty(session, setup_lib_db_with_score_set): assert get_allele_measurements(session, "CA000", user_data=_user_data(session)) == [] + + +def _oddspaths_classification(ratio): + """A transient functional classification carrying only an oddspaths ratio — enough for + ``classification_evidence_strength`` (which reads ``acmg_classification`` [None here] then the ratio) + to rank it, with no DB round-trip.""" + return ScoreCalibrationFunctionalClassification(oddspaths_ratio=ratio) + + +def _measurement(urn, *, relationship=MeasurementRelationship.direct, classification=None, is_current=True): + """A transit ``AlleleMeasurement`` with just the fields ``_ordering_key`` reads; the rest are filler.""" + return AlleleMeasurement( + variant_urn=urn, + score=None, + assay_level=None, + relationship=relationship, + assay_level_hgvs=None, + submitted_hgvs=None, + score_set_urn="", + score_set_title="", + primary_classification=classification, + is_current=is_current, + superseded_by_score_set=None, + ) + + +def test_ordering_key_ranks_every_dimension(): + """Pin the full precedence chain of ``_ordering_key`` as a pure unit (no DB, no calibration fixtures): + current → direct → classified → strongest magnitude → pathogenic direction → newest published → urn. + Each adjacent pair below differs by exactly the next-lower key, so sorting a shuffled copy must + reproduce this exact order.""" + strong_path = _oddspaths_classification(100.0) # magnitude |ln 100|, pathogenic + strong_benign = _oddspaths_classification(0.01) # equal magnitude, benign + weak_path = _oddspaths_classification(2.0) # smaller magnitude, pathogenic + + # (measurement, published_date), already in the expected ranked order. + ranked = [ + (_measurement("u#1", classification=strong_path), date(2024, 1, 1)), # strongest evidence, pathogenic + (_measurement("u#2", classification=strong_benign), date(2024, 1, 1)), # equal magnitude, benign loses + (_measurement("u#3", classification=weak_path), date(2024, 1, 1)), # weaker magnitude + (_measurement("u#4a", classification=None), date(2024, 6, 1)), # unclassified; newer published leads its tie + (_measurement("u#4b", classification=None), date(2024, 1, 1)), # unclassified, older published + (_measurement("u#4c", classification=None), date(2024, 1, 1)), # unclassified, same date → urn tiebreak + (_measurement("u#5", relationship=MeasurementRelationship.protein_consequence), date(2024, 6, 1)), # related + (_measurement("u#6", is_current=False), date(2024, 6, 1)), # superseded sorts last + ] + + shuffled = [ranked[i] for i in (5, 0, 7, 2, 4, 1, 6, 3)] + ordered = sorted(shuffled, key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == [m.variant_urn for m, _ in ranked] + + +def test_ordering_key_missing_published_date_sorts_after_dated(): + """A null published date is treated as oldest (``0`` ordinal), so a dated measurement leads an + otherwise-identical undated one.""" + dated = (_measurement("u#dated"), date(2024, 1, 1)) + undated = (_measurement("u#undated"), None) + + ordered = sorted([undated, dated], key=lambda pair: _ordering_key(pair[0], pair[1])) + + assert [m.variant_urn for m, _ in ordered] == ["u#dated", "u#undated"] diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 97d4fed43..8c01ec8ef 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -4,9 +4,12 @@ Exercise the HTTP surface: the equivalence-class list serializes and validates, the ``as_of`` content-time is echoed in ``X-As-Of``, an unknown id is an empty list (not a 404), superseded -measurements are opt-in, and a private score set's measurement never leaks. +measurements are opt-in, a private score set's measurement never leaks, and the response body's +order reflects the ranked sort (current-then-superseded, direct-then-related, newest-published, urn). """ +from datetime import date + import pytest arq = pytest.importorskip("arq") @@ -27,9 +30,10 @@ from tests.helpers.util.user import change_ownership -def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): +def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123", vrs_digest="cdna-digest"): """Give a variant a live coding-measured mapping record whose authoritative allele carries the - ClinGen id — the minimal shape for a direct measurement on that CA's page.""" + ClinGen id — the minimal shape for a direct measurement on that CA's page. ``vrs_digest`` is + content-unique (DB constraint), so pass a distinct one when seeding more than one per test.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -40,7 +44,7 @@ def _seed_cdna_measurement(session, variant_urn, *, clingen_allele_id="CA123"): session.add(record) session.commit() - measured = Allele(vrs_digest="cdna-digest", level="cdna", clingen_allele_id=clingen_allele_id) + measured = Allele(vrs_digest=vrs_digest, level="cdna", clingen_allele_id=clingen_allele_id) session.add(measured) session.commit() session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True)) @@ -140,6 +144,113 @@ def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_fil assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" +def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): + """The list is ordered, and direct measurements precede related ones in the body: a CA+siblings query + returns the directly-assayed change first, then the sibling nucleotide encoding.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + prot = Allele(vrs_digest="prot-digest", level="protein", clingen_allele_id="PA9") + session.add(prot) + session.commit() + + for suffix, caid, digest in ((1, "CA111", "cdna-1"), (2, "CA222", "cdna-2")): + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#{suffix}")) + record = MappingRecord(variant_id=variant.id, assay_level="cdna", mapping_api_version="test.0.0") + session.add(record) + session.commit() + nt = Allele(vrs_digest=digest, level="cdna", clingen_allele_id=caid) + session.add(nt) + session.commit() + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=nt.id, is_authoritative=True)) + session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) + session.commit() + + response = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["relationship"]) for m in body] == [ + (f"{score_set['urn']}#1", "direct"), + (f"{score_set['urn']}#2", "nucleotide_encoding"), + ] + + +def test_current_measurements_sort_before_superseded(client, session, data_provider, data_files, setup_router_db): + """With superseded measurements opted in, current ones sort ahead of superseded ones in the body.""" + experiment = create_experiment(client) + older = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + newer = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + older_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == older["urn"])) + newer_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == newer["urn"])) + newer_db.superseded_score_set_id = older_db.id + session.commit() + _seed_cdna_measurement(session, f"{older['urn']}#1", vrs_digest="cdna-older") + _seed_cdna_measurement(session, f"{newer['urn']}#1", vrs_digest="cdna-newer") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements", params={"include_superseded": True}) + + assert response.status_code == 200 + body = response.json() + assert [(m["variantUrn"], m["isCurrent"]) for m in body] == [ + (f"{newer['urn']}#1", True), + (f"{older['urn']}#1", False), + ] + + +def test_measurements_sort_newest_published_first(client, session, data_provider, data_files, setup_router_db): + """Between two otherwise-equal current, unclassified measurements the newest-published sorts first — + published date outranks the urn tiebreak (here the newer set has the later urn, yet still leads).""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + first_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == first["urn"])) + second_db = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == second["urn"])) + first_db.published_date = date(2024, 1, 1) + second_db.published_date = date(2024, 6, 1) # later-published, and later urn + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + assert [m["variantUrn"] for m in response.json()] == [f"{second['urn']}#1", f"{first['urn']}#1"] + + +def test_measurements_tiebreak_on_urn(client, session, data_provider, data_files, setup_router_db): + """With every other sort key equal (both current, direct, unclassified, same published date) the urn is + the final, stable tiebreak — ascending.""" + experiment = create_experiment(client) + first = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + second = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + for urn in (first["urn"], second["urn"]): + score_set = session.scalar(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == urn)) + score_set.published_date = date(2024, 1, 1) + session.commit() + _seed_cdna_measurement(session, f"{first['urn']}#1", vrs_digest="cdna-first") + _seed_cdna_measurement(session, f"{second['urn']}#1", vrs_digest="cdna-second") + + response = client.get("/api/v1/clingen-alleles/CA123/measurements") + + assert response.status_code == 200 + returned = [m["variantUrn"] for m in response.json()] + assert returned == sorted([f"{first['urn']}#1", f"{second['urn']}#1"]) + + def test_anonymous_cannot_see_private_measurement( client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides ): From 0ff6cd0ca1c728b88af71ac797c8f21ff2f5454a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sun, 12 Jul 2026 21:27:36 -0700 Subject: [PATCH 72/93] feat(clinical-controls): tag control links with annotating allele digest Reshape the score-set clinical-controls response around the allele-link substrate and off the legacy mapped-variant model. - Introduce ControlVariantLink in the clinical-controls lib, carrying the VRS digest of the allele each ClinVar control annotates so clients can apply D78 precedence against the variant's authoritative assay-level digest - Extract the response leaf model out of mapped_variant.py into clinical_control.py as ClinvarVariantLink, dropping its dead variant-URN coercion validator and unused from_attributes - Rename the container view models to ClinicalControlWithClinvarLinks (and Saved variant) and the field mapped_variants -> clinvar_links - Update the router builder and clinical-control tests, including coverage for two controls reaching one variant via distinct alleles --- src/mavedb/lib/clinical_controls.py | 45 +++++++++++---- src/mavedb/routers/score_sets.py | 12 ++-- src/mavedb/view_models/clinical_control.py | 17 ++++-- src/mavedb/view_models/mapped_variant.py | 16 ------ tests/helpers/constants.py | 8 +-- tests/routers/test_score_set.py | 65 +++++++++++++++++++--- 6 files changed, 115 insertions(+), 48 deletions(-) diff --git a/src/mavedb/lib/clinical_controls.py b/src/mavedb/lib/clinical_controls.py index 5aa10b1ce..b852ad39e 100644 --- a/src/mavedb/lib/clinical_controls.py +++ b/src/mavedb/lib/clinical_controls.py @@ -9,6 +9,7 @@ place. """ +from dataclasses import dataclass from datetime import datetime from typing import Any, Optional, TypeVar @@ -25,6 +26,21 @@ SelectT = TypeVar("SelectT", bound=Select[Any]) +@dataclass(frozen=True) +class ControlVariantLink: + """One (score-set variant, annotated allele) pair that a clinical control reaches. + + ``allele_digest`` is the VRS digest of the allele the control actually annotates — the entity + ClinVar made its call on. A client applies **D78 precedence** by comparing it to the variant's + authoritative ``assay_level_digest``: a match is a call *on the assayed level* (which wins), a + mismatch is a call on a *projection sibling* (the fallback that only fires when the assayed level + is unannotated). Without the digest the two are indistinguishable and precedence is unimplementable. + """ + + variant_urn: str + allele_digest: str + + def _scope_to_live_score_set_controls(stmt: SelectT, score_set_id: int, as_of: Optional[datetime]) -> SelectT: """Join ``stmt`` (already selecting from :class:`ClinvarControl`) down the allele-link chain to the score set's variants and constrain every ``ValidTime`` hop to ``as_of``. ``distinct`` because the @@ -50,12 +66,15 @@ def get_clinical_controls_with_variant_urns( as_of: Optional[datetime] = None, db_name: Optional[str] = None, db_version: Optional[str] = None, -) -> list[tuple[ClinvarControl, list[str]]]: - """The live ClinVar controls for a score set, each paired with the URNs of the score-set variants - that link to it. Optionally narrowed to one control DB (``db_name``) and/or release (``db_version``). - Controls come back in first-seen order; each control's variant URNs preserve query order.""" +) -> list[tuple[ClinvarControl, list[ControlVariantLink]]]: + """The live ClinVar controls for a score set, each paired with the score-set variants that link to + it — every link carrying the digest of the allele the control annotates (see + :class:`ControlVariantLink`). Optionally narrowed to one control DB (``db_name``) and/or release + (``db_version``). Controls come back in first-seen order; each control's links preserve query order.""" stmt = _scope_to_live_score_set_controls( - select(ClinvarControl, Variant.urn.label("variant_urn")), score_set_id, as_of + select(ClinvarControl, Variant.urn.label("variant_urn"), Allele.vrs_digest.label("allele_digest")), + score_set_id, + as_of, ) if db_name is not None: stmt = stmt.where(ClinvarControl.db_name == db_name) @@ -63,18 +82,20 @@ def get_clinical_controls_with_variant_urns( stmt = stmt.where(ClinvarControl.db_version == db_version) controls: dict[int, ClinvarControl] = {} - urns_by_control: dict[int, list[str]] = {} - for ctrl, variant_urn in db.execute(stmt).tuples(): - # A persisted variant reached through the join always carries a URN (unique, set at creation); - # narrow the nullable column so the URN list matches the view model's required `str`. + links_by_control: dict[int, list[ControlVariantLink]] = {} + + # A persisted variant reached through the join always carries a URN (unique, set at creation); + # narrow the nullable column so the URN matches the view model's required `str`. + for ctrl, variant_urn, allele_digest in db.execute(stmt).tuples(): if variant_urn is None: continue + if ctrl.id not in controls: controls[ctrl.id] = ctrl - urns_by_control[ctrl.id] = [] - urns_by_control[ctrl.id].append(variant_urn) + links_by_control[ctrl.id] = [] + links_by_control[ctrl.id].append(ControlVariantLink(variant_urn=variant_urn, allele_digest=allele_digest)) - return [(ctrl, urns_by_control[ctrl.id]) for ctrl in controls.values()] + return [(ctrl, links_by_control[ctrl.id]) for ctrl in controls.values()] def get_clinical_control_options( diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 413d64a07..7a68e8550 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -2440,7 +2440,7 @@ async def publish_score_set( @router.get( "/score-sets/{urn}/clinical-controls", status_code=200, - response_model=list[clinical_control.ClinicalControlWithMappedVariants], + response_model=list[clinical_control.ClinicalControlWithClinvarLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get clinical controls for a score set", @@ -2461,7 +2461,7 @@ async def get_clinical_controls_for_score_set( user_data: UserData = Depends(get_current_user), db: Optional[str] = None, version: Optional[str] = None, -) -> list[clinical_control.ClinicalControlWithMappedVariants]: +) -> list[clinical_control.ClinicalControlWithClinvarLinks]: """ Fetch relevant clinical controls for a given score set. """ @@ -2503,7 +2503,7 @@ async def get_clinical_controls_for_score_set( save_to_logging_context({"resource_count": len(controls)}) return [ - clinical_control.ClinicalControlWithMappedVariants.model_validate( + clinical_control.ClinicalControlWithClinvarLinks.model_validate( { "id": ctrl.id, "db_identifier": ctrl.db_identifier, @@ -2514,10 +2514,12 @@ async def get_clinical_controls_for_score_set( "db_name": ctrl.db_name, "modification_date": ctrl.modification_date, "creation_date": ctrl.creation_date, - "mapped_variants": [{"variant_urn": v_urn} for v_urn in variant_urns], + "clinvar_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], } ) - for ctrl, variant_urns in controls + for ctrl, links in controls ] diff --git a/src/mavedb/view_models/clinical_control.py b/src/mavedb/view_models/clinical_control.py index 85cd78348..7e6a17d06 100644 --- a/src/mavedb/view_models/clinical_control.py +++ b/src/mavedb/view_models/clinical_control.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariantCreate, MappedVariantForClinicalControl + from mavedb.view_models.mapped_variant import MappedVariantCreate class ClinicalControlBase(BaseModel): @@ -41,8 +41,17 @@ class Config: from_attributes = True -class SavedClinicalControlWithMappedVariants(SavedClinicalControl): - mapped_variants: Sequence["MappedVariantForClinicalControl"] +class ClinvarVariantLink(BaseModel): + """One score-set variant a ClinVar control reaches, tagged with the annotated allele's digest.""" + + variant_urn: str + # VRS digest of the allele this control annotates. Many controls + # may share the same variant_urn, but each will have a different allele_digest. + allele_digest: Optional[str] = None + + +class SavedClinicalControlWithClinvarLinks(SavedClinicalControl): + clinvar_links: Sequence[ClinvarVariantLink] # Properties to return to non-admin clients @@ -50,7 +59,7 @@ class ClinicalControl(SavedClinicalControl): pass -class ClinicalControlWithMappedVariants(SavedClinicalControlWithMappedVariants): +class ClinicalControlWithClinvarLinks(SavedClinicalControlWithClinvarLinks): pass diff --git a/src/mavedb/view_models/mapped_variant.py b/src/mavedb/view_models/mapped_variant.py index 218f0ba8d..7c03fd103 100644 --- a/src/mavedb/view_models/mapped_variant.py +++ b/src/mavedb/view_models/mapped_variant.py @@ -111,22 +111,6 @@ class MappedVariantWithControls(SavedMappedVariantWithControls): gnomad_variants: Sequence["GnomADVariant"] -class MappedVariantForClinicalControl(BaseModel): - variant_urn: str - - class Config: - from_attributes = True - - @model_validator(mode="before") - def generate_score_set_urn_list(cls, data: Any): - if not hasattr(data, "variant_urn") and hasattr(data, "variant"): - try: - data.__setattr__("variant_urn", None if not data.variant else data.variant.urn) - except AttributeError as exc: - raise ValidationError(f"Unable to create {cls.__name__} without attribute: {exc}.") # type: ignore - return data - - # ruff: noqa: E402 from mavedb.view_models.clinical_control import ClinicalControl, ClinicalControlBase, SavedClinicalControl from mavedb.view_models.gnomad_variant import GnomADVariant, GnomADVariantBase, SavedGnomADVariant diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index bf78d38fe..8766c2f53 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1781,14 +1781,14 @@ TEST_SAVED_CLINVAR_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "183058", "geneSymbol": "PTEN", "clinicalSignificance": "Likely benign", "clinicalReviewStatus": "criteria provided, multiple submitters, no conflicts", "dbName": "ClinVar", "dbVersion": "11_2024", - "mappedVariants": [], + "clinvarLinks": [], } @@ -1803,14 +1803,14 @@ TEST_SAVED_GENERIC_CLINICAL_CONTROL = { - "recordType": "ClinicalControlWithMappedVariants", + "recordType": "ClinicalControlWithClinvarLinks", "dbIdentifier": "ABC123", "geneSymbol": "BRCA1", "clinicalSignificance": "benign", "clinicalReviewStatus": "lots of convincing evidence", "dbName": "GenDB", "dbVersion": "2024", - "mappedVariants": [], + "clinvarLinks": [], } diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 632c0fa15..942f032c7 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -2892,9 +2892,7 @@ def test_search_score_sets_not_affected_by_experiment_metadata( assert response.json()["numScoreSets"] == num_score_sets -def test_cannot_create_multiple_superseding_versions( - session, data_provider, client, setup_router_db, data_files -): +def test_cannot_create_multiple_superseding_versions(session, data_provider, client, setup_router_db, data_files): """Attempting to create multiple superseding versions should fail.""" experiment = create_experiment(client, {"title": "Original Experiment"}) score_set = create_seq_score_set(client, experiment["urn"], update={"title": "Original Score Set"}) @@ -2920,7 +2918,9 @@ def test_cannot_create_multiple_superseding_versions( response = client.post("/api/v1/score-sets/", json=score_set_post_payload) assert response.status_code == 409 - assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()["detail"] + assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()[ + "detail" + ] def test_search_score_sets_not_affected_by_an_unpublishing_superseding_versions( @@ -3867,15 +3867,66 @@ def test_can_fetch_current_clinical_controls_for_score_set(client, setup_router_ response_data = response.json() assert len(response_data) == 2 for control in response_data: - mapped_variants = control.pop("mappedVariants") - assert len(mapped_variants) == 1 + clinvar_links = control.pop("clinvarLinks") + assert len(clinvar_links) == 1 + # Each link carries the digest of the allele the control annotates — the D78 precedence signal. + assert clinvar_links[0]["alleleDigest"] in ("clinical-control-allele-0", "clinical-control-allele-1") assert all( control[k] in (TEST_SAVED_CLINVAR_CONTROL[k], TEST_SAVED_GENERIC_CLINICAL_CONTROL[k]) for k in TEST_SAVED_CLINVAR_CONTROL.keys() - if k != "mappedVariants" + if k != "clinvarLinks" ) +def test_clinical_controls_tag_each_link_with_the_annotating_allele_digest( + client, setup_router_db, session, data_provider, data_files +): + """A protein-change variant whose DNA siblings carry conflicting ClinVar calls surfaces *both* + controls against the *same* variant URN — distinguishable only by the digest of the allele each + one annotates. That digest is D78's precedence signal: the control on the variant's authoritative + (assayed-level) allele is the direct call; the control on a projection sibling is the fallback. + Without it the two collapse to one variant and precedence is unimplementable.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = session.scalars( + select(VariantDbModel.urn) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).first() + + # One variant, two alleles in its equivalence class: the assayed (authoritative) allele carries the + # ClinVar control (id 1); a projection sibling carries the generic control (id 2). + seed_mapping_record( + session, + variant_urn, + assay_level="protein", + alleles=[ + AlleleSpec(digest="assayed-level-allele", level="protein", is_authoritative=True, clinvar_control_ids=[1]), + AlleleSpec( + digest="projection-sibling-allele", level="cdna", is_authoritative=False, clinvar_control_ids=[2] + ), + ], + ) + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/clinical-controls") + assert response.status_code == 200 + + controls = {c["dbIdentifier"]: c for c in response.json()} + assert len(controls) == 2 + + assayed = controls[TEST_SAVED_CLINVAR_CONTROL["dbIdentifier"]] # control 1, on the authoritative allele + sibling = controls[TEST_SAVED_GENERIC_CLINICAL_CONTROL["dbIdentifier"]] # control 2, on the sibling + + # Both controls point at the same variant, so only the annotating allele's digest tells them apart. + assert [link["variantUrn"] for link in assayed["clinvarLinks"]] == [variant_urn] + assert [link["variantUrn"] for link in sibling["clinvarLinks"]] == [variant_urn] + assert assayed["clinvarLinks"][0]["alleleDigest"] == "assayed-level-allele" + assert sibling["clinvarLinks"][0]["alleleDigest"] == "projection-sibling-allele" + + @pytest.mark.parametrize("clinical_control", [TEST_SAVED_CLINVAR_CONTROL, TEST_SAVED_GENERIC_CLINICAL_CONTROL]) @pytest.mark.parametrize( "parameters", [[("db", "dbName")], [("version", "dbVersion")], [("db", "dbName"), ("version", "dbVersion")]] From bb369d9e30c1f6122472e341a57a94c17179a63b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 13 Jul 2026 13:33:51 -0700 Subject: [PATCH 73/93] refactor(allele-measurements): expose preferred classification, exclude RUO Rename the measurement's inline classification field from ``primary_classification``/``primaryClassification`` to ``preferred_classification``/``preferredClassification`` to reflect that the surfaced value is the UI's default from the primary-first preference cascade, not necessarily the promoted primary calibration. - Exclude research-use-only calibrations outright from the preferred classification on this clinical surface, rather than merely deprioritizing them as the shared cascade does - Add a test covering RUO exclusion and fix a self-contradictory cascade fixture that marked a calibration primary while asserting an investigator-provided one should win --- src/mavedb/lib/allele_measurements.py | 28 ++++++++++------- src/mavedb/view_models/allele_measurement.py | 7 +++-- tests/lib/test_allele_measurements.py | 33 +++++++++++++++----- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 9419ea244..6ca2a6d23 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -70,8 +70,8 @@ class AlleleMeasurement: ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried - ClinGen id. ``primary_classification`` is the primary readable functional classification (``None`` - when the score set has none or the caller cannot read the calibration). + ClinGen id. ``preferred_classification`` is the readable functional classification the UI defaults to + (primary-first cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). """ variant_urn: str @@ -82,7 +82,7 @@ class AlleleMeasurement: submitted_hgvs: Optional[str] score_set_urn: str score_set_title: str - primary_classification: Optional[ScoreCalibrationFunctionalClassification] + preferred_classification: Optional[ScoreCalibrationFunctionalClassification] is_current: bool superseded_by_score_set: Optional[str] @@ -94,12 +94,15 @@ def _preferred_classification( A variant falls into one classification per calibration (non-overlapping ranges); we surface the one the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then - ``investigator_provided``, then non-``research_use_only``. Within a tier we take the strongest - evidence (the VA-Spec posture — the strongest calibration is the evidence calibration), then ``id`` - only for determinism. The UI's "any with ranges / any calibration" tail steps collapse here: this - query only surfaces calibrations that already classify the variant. Only calibrations the caller can - read (calibration READ) are considered, so a private calibration is skipped rather than blanking a - readable call. Kept in sync with the UI's `activeCalibrationOptions` default selection. + ``investigator_provided``. Within a tier we take the strongest evidence (the VA-Spec posture — the + strongest calibration is the evidence calibration), then ``id`` only for determinism. + + This is a clinical surface, so research-use-only calibrations are excluded outright (not merely deprioritized + as in the shared cascade) — a RUO call is never the classification shown here. + + Only calibrations the caller can read (calibration READ) are considered, so a private calibration is + skipped rather than blanking a readable call. Kept in sync with the UI's `activeCalibrationOptions` + default selection. """ candidates = [ (classification, calibration) @@ -111,6 +114,7 @@ def _preferred_classification( ) .join(ScoreCalibration, ScoreCalibration.id == ScoreCalibrationFunctionalClassification.calibration_id) .where(classification_variants.c.variant_id == variant.id) + .where(ScoreCalibration.research_use_only.is_(False)) ).all() if has_permission(user_data, calibration, Action.READ).permitted ] @@ -137,11 +141,11 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] 4. Within each, the newest-published first 5. Within each, the URN for a stable tiebreak """ - magnitude, direction = classification_evidence_strength(measurement.primary_classification) + magnitude, direction = classification_evidence_strength(measurement.preferred_classification) return ( 0 if measurement.is_current else 1, 0 if measurement.relationship == MeasurementRelationship.direct else 1, - 0 if measurement.primary_classification is not None else 1, + 0 if measurement.preferred_classification is not None else 1, -magnitude, direction, -(published_date.toordinal()) if published_date is not None else 0, @@ -257,7 +261,7 @@ def get_allele_measurements( submitted_hgvs=variant.hgvs_pro if assay_level == SequenceLevel.protein.value else variant.hgvs_nt, score_set_urn=score_set.urn or "", score_set_title=score_set.title or "", - primary_classification=_preferred_classification(db, variant, user_data=user_data), + preferred_classification=_preferred_classification(db, variant, user_data=user_data), is_current=is_current, superseded_by_score_set=superseding.urn if superseding is not None else None, ) diff --git a/src/mavedb/view_models/allele_measurement.py b/src/mavedb/view_models/allele_measurement.py index 592285d12..0baebaddd 100644 --- a/src/mavedb/view_models/allele_measurement.py +++ b/src/mavedb/view_models/allele_measurement.py @@ -20,8 +20,9 @@ class AlleleMeasurement(BaseModel): ``genomic``) — always shown, since the measured level is the clinically load-bearing fact. ``relationship`` says how the measurement relates to the queried ClinGen id: ``direct`` (assayed at this allele), ``protein_consequence`` (a protein measurement of a nt query's consequence), or - ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``primaryClassification`` is the - primary readable functional classification, omitted when absent or gated. ``isCurrent`` / + ``nucleotide_encoding`` (a nt measurement encoding a protein query). ``preferredClassification`` is the + readable functional classification the UI defaults to (primary-first cascade, RUO excluded), omitted + when absent or gated. ``isCurrent`` / ``supersededByScoreSet`` let a superseded measurement (surfaced only under ``include_superseded``) self-describe; ``supersededByScoreSet`` is the superseding *score set*'s URN. """ @@ -34,7 +35,7 @@ class AlleleMeasurement(BaseModel): submitted_hgvs: Optional[str] = None score_set_urn: str score_set_title: str - primary_classification: Optional[SavedFunctionalClassification] = None + preferred_classification: Optional[SavedFunctionalClassification] = None is_current: bool superseded_by_score_set: Optional[str] = None diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index d64b33e6a..a4621c33b 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -294,15 +294,15 @@ def test_calibration_gate_withholds_classification(session, setup_lib_db_with_sc anonymous = get_allele_measurements(session, "CA123", user_data=None) assert len(anonymous) == 1 - assert anonymous[0].primary_classification is None + assert anonymous[0].preferred_classification is None owner = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert owner[0].primary_classification is not None - assert owner[0].primary_classification.functional_classification.value == "abnormal" + assert owner[0].preferred_classification is not None + assert owner[0].preferred_classification.functional_classification.value == "abnormal" @pytest.mark.integration -def test_primary_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): +def test_preferred_classification_cascade_beats_strength(session, setup_lib_db_with_score_set): """With no primary calibration, the preference cascade dominates evidence strength: an investigator-provided calibration wins over a stronger non-RUO one and a stronger-still RUO one.""" score_set = setup_lib_db_with_score_set @@ -317,11 +317,11 @@ def test_primary_classification_cascade_beats_strength(session, setup_lib_db_wit result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) # Investigator-provided (weakest) is chosen over the stronger non-RUO and RUO calibrations. - assert result[0].primary_classification.oddspaths_ratio == 2.0 + assert result[0].preferred_classification.oddspaths_ratio == 2.0 @pytest.mark.integration -def test_primary_classification_strongest_within_tier(session, setup_lib_db_with_score_set): +def test_preferred_classification_strongest_within_tier(session, setup_lib_db_with_score_set): """Within one cascade tier (here two plain non-RUO calibrations), the strongest evidence wins.""" score_set = setup_lib_db_with_score_set nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") @@ -333,7 +333,24 @@ def test_primary_classification_strongest_within_tier(session, setup_lib_db_with result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) - assert result[0].primary_classification.oddspaths_ratio == 100.0 + assert result[0].preferred_classification.oddspaths_ratio == 100.0 + + +@pytest.mark.integration +def test_research_use_only_calibrations_are_excluded(session, setup_lib_db_with_score_set): + """A research-use-only calibration is excluded from the cascade (even if it is the strongest + evidence).""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + variant = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + _link(session, _record(session, variant, assay_level="cdna"), nt, is_authoritative=True) + + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=100.0, research_use_only=True) + _calibration(session, score_set, variants=[variant], primary=False, oddspaths_ratio=1000.0, research_use_only=True) + + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + + assert result[0].preferred_classification is None @pytest.mark.integration @@ -429,7 +446,7 @@ def _measurement(urn, *, relationship=MeasurementRelationship.direct, classifica submitted_hgvs=None, score_set_urn="", score_set_title="", - primary_classification=classification, + preferred_classification=classification, is_current=is_current, superseded_by_score_set=None, ) From c1fb812b0065513438fd3732cfd0bf430e6bd7d9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 15 Jul 2026 09:42:47 -0700 Subject: [PATCH 74/93] feat(clingen): batch CAR submissions and add HTTP timeouts Reverse translation can produce very large allele counts, so the CAR submission now chunks HGVS into fixed-size batches instead of a single PUT. Each batch is dispatched and reconciled independently, so a transient failure or a broken one-result-per-input response fails only that batch's alleles rather than the whole run. - add DEFAULT_CAR_SUBMISSION_BATCH_SIZE (env-configurable) and loop the dispatch/reconcile logic per batch, scaling progress across chunks - add an explicit (connect, read) timeout to all ClinGen HTTP calls so a stalled server can no longer hang the worker indefinitely; widen the LDH authenticate catch to RequestException - cover batch splitting and per-batch failure isolation with tests --- src/mavedb/lib/clingen/constants.py | 9 + src/mavedb/lib/clingen/services.py | 12 +- .../worker/jobs/external_services/clingen.py | 278 ++++++++++-------- .../jobs/external_services/test_clingen.py | 165 +++++++++++ 4 files changed, 332 insertions(+), 132 deletions(-) diff --git a/src/mavedb/lib/clingen/constants.py b/src/mavedb/lib/clingen/constants.py index 2276ee187..60f3a26cb 100644 --- a/src/mavedb/lib/clingen/constants.py +++ b/src/mavedb/lib/clingen/constants.py @@ -14,8 +14,17 @@ LDH_ENTITY_ENDPOINT = "maveDb" # for some reason, not the same :/ DEFAULT_LDH_SUBMISSION_BATCH_SIZE = 100 +DEFAULT_CAR_SUBMISSION_BATCH_SIZE = int(os.getenv("CAR_SUBMISSION_BATCH_SIZE", "10000")) +"""Number of HGVS strings sent to the ClinGen Allele Registry per PUT. Reverse translation can +produce very large allele counts, so submissions are chunked rather than sent as a single payload.""" CLINGEN_CACHE_WARMING_CONCURRENCY = int(os.getenv("CLINGEN_CACHE_WARMING_CONCURRENCY", "5")) """Maximum number of concurrent requests to make to the ClinGen API when pre-warming the cache for mapped variants.""" + +CLINGEN_CONNECT_TIMEOUT = float(os.getenv("CLINGEN_CONNECT_TIMEOUT", "5")) +CLINGEN_READ_TIMEOUT = float(os.getenv("CLINGEN_READ_TIMEOUT", "300")) +CLINGEN_HTTP_TIMEOUT = (CLINGEN_CONNECT_TIMEOUT, CLINGEN_READ_TIMEOUT) +"""(connect, read) timeout for ClinGen HTTP calls. Fail fast on connect, with a generous read +budget because a large batch can take CAR a while to respond.""" LDH_SUBMISSION_ENDPOINT = f"https://genboree.org/mq/brdg/pulsar/{CLIN_GEN_TENANT}/ldh/submissions/{LDH_ENTITY_ENDPOINT}" LDH_ACCESS_ENDPOINT = os.getenv("LDH_ACCESS_ENDPOINT", "https://genboree.org/ldh") LDH_MAVE_ACCESS_ENDPOINT = f"{LDH_ACCESS_ENDPOINT}/{LDH_ENTITY_NAME}/id" diff --git a/src/mavedb/lib/clingen/services.py b/src/mavedb/lib/clingen/services.py index 085d9103f..c8e80259f 100644 --- a/src/mavedb/lib/clingen/services.py +++ b/src/mavedb/lib/clingen/services.py @@ -8,7 +8,7 @@ import requests from jose import jwt -from mavedb.lib.clingen.constants import GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD +from mavedb.lib.clingen.constants import CLINGEN_HTTP_TIMEOUT, GENBOREE_ACCOUNT_NAME, GENBOREE_ACCOUNT_PASSWORD from mavedb.lib.logging.context import format_raised_exception_info_as_dict, logging_context, save_to_logging_context from mavedb.lib.types.clingen import ClinGenAllele, ClinGenSubmissionError, LdhSubmission from mavedb.lib.utils import batched @@ -80,6 +80,7 @@ def dispatch_submissions( response = requests.put( url=request_url, data="\n".join(content_submissions), + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() @@ -165,10 +166,10 @@ def authenticate(self) -> str: auth_url = f"https://genboree.org/auth/usr/gb:{GENBOREE_ACCOUNT_NAME}/auth" auth_body = {"type": "plain", "val": GENBOREE_ACCOUNT_PASSWORD} - auth_response = requests.post(auth_url, json=auth_body) try: + auth_response = requests.post(auth_url, json=auth_body, timeout=CLINGEN_HTTP_TIMEOUT) auth_response.raise_for_status() - except requests.exceptions.HTTPError as exc: + except requests.exceptions.RequestException as exc: save_to_logging_context(format_raised_exception_info_as_dict(exc)) logger.error(msg="Failed to authenticate with Genboree services.", exc_info=exc, extra=logging_context()) raise exc @@ -221,16 +222,17 @@ def dispatch_submissions( logger.info(msg=f"Dispatching {len(submissions)} ldh submissions...", extra=logging_context()) for idx, content in enumerate(submissions): try: - logger.debug(msg=f"Dispatching submission {idx+1}.", extra=logging_context()) + logger.debug(msg=f"Dispatching submission {idx + 1}.", extra=logging_context()) response = requests.put( url=self.url, json=content, headers={"Authorization": f"Bearer {self.authenticate()}", "Content-Type": "application/json"}, + timeout=CLINGEN_HTTP_TIMEOUT, ) response.raise_for_status() submission_successes.append(response.json()) logger.info( - msg=f"Successfully dispatched ldh submission ({idx+1} / {len(submissions)}).", + msg=f"Successfully dispatched ldh submission ({idx + 1} / {len(submissions)}).", extra=logging_context(), ) diff --git a/src/mavedb/worker/jobs/external_services/clingen.py b/src/mavedb/worker/jobs/external_services/clingen.py index e5a438ee1..d563e658c 100644 --- a/src/mavedb/worker/jobs/external_services/clingen.py +++ b/src/mavedb/worker/jobs/external_services/clingen.py @@ -21,6 +21,7 @@ from mavedb.lib.clingen.constants import ( CAR_SUBMISSION_ENDPOINT, CLIN_GEN_SUBMISSION_ENABLED, + DEFAULT_CAR_SUBMISSION_BATCH_SIZE, DEFAULT_LDH_SUBMISSION_BATCH_SIZE, LDH_SUBMISSION_ENDPOINT, ) @@ -30,6 +31,7 @@ ClinGenLdhService, ) from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.lib.utils import batched from mavedb.lib.variants import get_hgvs_from_post_mapped from mavedb.models.allele import Allele as AlleleModel from mavedb.models.enums.annotation_type import AnnotationType @@ -113,12 +115,14 @@ async def submit_score_set_mappings_to_car(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore force_reregister = bool(job.job_params.get("force_reregister", False)) # type: ignore[union-attr] - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_car", - "resource": score_set.urn, - "correlation_id": correlation_id, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_car", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) job_manager.update_progress(0, 100, "Starting CAR mapped resource submission.") logger.info(msg="Started CAR mapped resource submission", extra=job_manager.logging_context()) @@ -276,9 +280,6 @@ def _outcome_data() -> dict[str, int]: job_manager.update_progress(15, 100, "Submitting alleles to CAR.") car_service = ClinGenAlleleRegistryService(url=CAR_SUBMISSION_ENDPOINT) - hgvs_list = list(hgvs_to_allele_ids.keys()) - registered_alleles = car_service.dispatch_submissions(hgvs_list) - job_manager.update_progress(60, 100, "Processing registered alleles from CAR.") # Bulk-load every allele that could be linked in one query, keyed by id, rather than # issuing a SELECT per CAID inside the response loop. @@ -287,133 +288,153 @@ def _outcome_data() -> dict[str, int]: for allele in job_manager.db.scalars(select(AlleleModel).where(AlleleModel.id.in_(submitted_allele_ids))).all() } - # CAR's contract is one result per submitted HGVS, in order — that is what makes the - # positional zip below valid. A different count (including the empty list dispatch returns - # on request failure) means alignment cannot be trusted for ANY position, so we register - # nothing and fail the whole batch rather than risk writing a CAID to the wrong allele. - aligned = len(registered_alleles) == len(hgvs_list) - if not aligned: - logger.error( - msg=( - f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " - "positional alignment cannot be trusted. Failing the entire batch." - ), - extra=job_manager.logging_context(), - ) - - for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): - allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] + # Reverse translation can push allele counts high enough that a single PUT is untenable, so + # dispatch in fixed-size chunks. Each batch is reconciled independently: a transient failure + # (or a broken one-result-per-input contract) fails only that batch's alleles, not the run. + batches = list(batched(list(hgvs_to_allele_ids.keys()), DEFAULT_CAR_SUBMISSION_BATCH_SIZE)) + job_manager.save_to_context( + { + "car_submission_batch_size": DEFAULT_CAR_SUBMISSION_BATCH_SIZE, + "car_submission_batch_count": len(batches), + } + ) - if "errorType" in response: - logger.warning( - msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + for batch_idx, hgvs_list in enumerate(batches): + # Spread the 15→60 window across batches so progress advances as each chunk is dispatched. + job_manager.update_progress( + 15 + (45 * batch_idx) // len(batches), + 100, + f"Submitting alleles to CAR (batch {batch_idx + 1} / {len(batches)}).", + ) + registered_alleles = car_service.dispatch_submissions(hgvs_list) + + # CAR's contract is one result per submitted HGVS, in order — that is what makes the + # positional zip below valid. A different count (including the empty list dispatch returns + # on request failure) means alignment cannot be trusted for ANY position, so we register + # nothing and fail this batch rather than risk writing a CAID to the wrong allele. + aligned = len(registered_alleles) == len(hgvs_list) + if not aligned: + logger.error( + msg=( + f"CAR returned {len(registered_alleles)} results for {len(hgvs_list)} submitted HGVS; " + "positional alignment cannot be trusted. Failing this batch." + ), extra=job_manager.logging_context(), ) - for allele_id in allele_ids_for_hgvs: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.SERVICE_REJECTED, - error_message="Failed to register allele with ClinGen Allele Registry.", - metadata={ - "submitted_hgvs": hgvs_string, - "car_error_type": response.get("errorType"), - "car_error_message": response.get("message"), - }, - ) - continue + for hgvs_string, response in zip(hgvs_list, registered_alleles if aligned else []): + allele_ids_for_hgvs = hgvs_to_allele_ids[hgvs_string] - # A response that is neither an error nor a registration (no "@id") is malformed. - caid_iri = response.get("@id") - if not caid_iri: - logger.error( - msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", - extra=job_manager.logging_context(), - ) - for allele_id in allele_ids_for_hgvs: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.MALFORMED_RESPONSE, - error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", - metadata={"submitted_hgvs": hgvs_string}, + if "errorType" in response: + logger.warning( + msg=f"CAR rejected HGVS '{hgvs_string}' ({response.get('errorType', 'unknown')}): {response.get('message', 'unknown')}", + extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.SERVICE_REJECTED, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={ + "submitted_hgvs": hgvs_string, + "car_error_type": response.get("errorType"), + "car_error_message": response.get("message"), + }, + ) - continue - - caid = caid_iri.split("/")[-1] - for allele_id in allele_ids_for_hgvs: - entry = allele_data[allele_id] - prior_caid = entry.existing_caid + continue - # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant - # violation. Do not overwrite; record a failure with full audit context. - if prior_caid and prior_caid != caid: + # A response that is neither an error nor a registration (no "@id") is malformed. + caid_iri = response.get("@id") + if not caid_iri: logger.error( - msg=( - f"CAR returned a different CAID for allele {allele_id}: " - f"stored={prior_caid!r}, returned={caid!r}. " - "Not overwriting. Investigate immediately." - ), + msg=f"CAR returned a response for HGVS '{hgvs_string}' with neither an error nor an allele identifier.", extra=job_manager.logging_context(), ) + for allele_id in allele_ids_for_hgvs: + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.MALFORMED_RESPONSE, + error_message="ClinGen Allele Registry returned a malformed response with no allele identifier.", + metadata={"submitted_hgvs": hgvs_string}, + ) - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.CAID_CONFLICT, - error_message="CAR returned a CAID that conflicts with the stored value.", - metadata={ - "clingen_allele_id": prior_caid, - "conflicting_caid": caid, - "submitted_hgvs": hgvs_string, - }, - ) - - # CAID is new or matches the stored value — link it to the allele and record success. - else: - linked_allele_ids.add(allele_id) - allele = alleles_by_id[allele_id] - allele.clingen_allele_id = caid + continue - reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED - if prior_caid: - logger.info( - msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + caid = caid_iri.split("/")[-1] + for allele_id in allele_ids_for_hgvs: + entry = allele_data[allele_id] + prior_caid = entry.existing_caid + + # TODO#780 - CAID is immutable — a different value returned by CAR is a hard invariant + # violation. Do not overwrite; record a failure with full audit context. + if prior_caid and prior_caid != caid: + logger.error( + msg=( + f"CAR returned a different CAID for allele {allele_id}: " + f"stored={prior_caid!r}, returned={caid!r}. " + "Not overwriting. Investigate immediately." + ), extra=job_manager.logging_context(), ) + failed_allele_ids.add(allele_id) + _annotate_caid( + annotation_manager, + allele_id, + Disposition.FAILED, + EventReason.CAID_CONFLICT, + error_message="CAR returned a CAID that conflicts with the stored value.", + metadata={ + "clingen_allele_id": prior_caid, + "conflicting_caid": caid, + "submitted_hgvs": hgvs_string, + }, + ) + + # CAID is new or matches the stored value — link it to the allele and record success. + else: + linked_allele_ids.add(allele_id) + allele = alleles_by_id[allele_id] + allele.clingen_allele_id = caid + + reason = EventReason.RECONFIRMED if prior_caid else EventReason.CREATED + if prior_caid: + logger.info( + msg=f"Force re-registration confirmed same CAID {caid!r} for allele {allele_id}.", + extra=job_manager.logging_context(), + ) + + _annotate_caid( + annotation_manager, + allele_id, + Disposition.PRESENT, + reason, + metadata={"clingen_allele_id": caid}, + ) + + # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up + # (network drop, service-side omission), or the entire batch when the response count + # violated CAR's one-result-per-input contract and we rejected it above. + unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] + for hgvs_string in unattributed_hgvs: + for allele_id in hgvs_to_allele_ids[hgvs_string]: + failed_allele_ids.add(allele_id) _annotate_caid( annotation_manager, allele_id, - Disposition.PRESENT, - reason, - metadata={"clingen_allele_id": caid}, + Disposition.FAILED, + EventReason.API_ERROR, + error_message="Failed to register allele with ClinGen Allele Registry.", + metadata={"submitted_hgvs": hgvs_string}, ) - # Submitted HGVS with no trustworthy response: the truncated tail when the counts line up - # (network drop, service-side omission), or the entire batch when the response count - # violated CAR's one-result-per-input contract and we rejected it above. - unattributed_hgvs = hgvs_list if not aligned else hgvs_list[len(registered_alleles) :] - for hgvs_string in unattributed_hgvs: - for allele_id in hgvs_to_allele_ids[hgvs_string]: - failed_allele_ids.add(allele_id) - _annotate_caid( - annotation_manager, - allele_id, - Disposition.FAILED, - EventReason.API_ERROR, - error_message="Failed to register allele with ClinGen Allele Registry.", - metadata={"submitted_hgvs": hgvs_string}, - ) - annotation_manager.flush() outcome_data = _outcome_data() @@ -477,12 +498,14 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: correlation_id = job.job_params["correlation_id"] # type: ignore # Setup initial context and progress - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "submit_score_set_mappings_to_ldh", - "resource": score_set.urn, - "correlation_id": correlation_id, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "submit_score_set_mappings_to_ldh", + "resource": score_set.urn, + "correlation_id": correlation_id, + } + ) job_manager.update_progress(0, 100, "Starting LDH mapped resource submission.") logger.info(msg="Started LDH mapped resource submission", extra=job_manager.logging_context()) @@ -496,8 +519,7 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: # MaveDB score to its canonical mapped variant, not to every equivalent allele (unlike CAR, # which registers a CAID per allele). variant_objects: Sequence[tuple[Variant, MappingRecord, AlleleModel]] = ( - job_manager.db - .execute( + job_manager.db.execute( select(Variant, MappingRecord, AlleleModel) .join(MappingRecord, MappingRecord.variant_id == Variant.id) .join(MappingRecordAllele, MappingRecordAllele.mapping_record_id == MappingRecord.id) @@ -560,10 +582,12 @@ async def submit_score_set_mappings_to_ldh(ctx: dict, job_id: int, job_manager: loop = asyncio.get_running_loop() submission_successes, submission_failures = await loop.run_in_executor(ctx["pool"], blocking) job_manager.update_progress(90, 100, "Finalizing LDH mapped resource submission.") - job_manager.save_to_context({ - "ldh_submission_successes": len(submission_successes), - "ldh_submission_failures": len(submission_failures), - }) + job_manager.save_to_context( + { + "ldh_submission_successes": len(submission_successes), + "ldh_submission_failures": len(submission_failures), + } + ) # See https://reg.clinicalgenome.org/doc/AlleleRegistry_1.01.xx_api_v1.pdf annotation_manager = AnnotationStatusManager( diff --git a/tests/worker/jobs/external_services/test_clingen.py b/tests/worker/jobs/external_services/test_clingen.py index 3b44a35c5..5431227d6 100644 --- a/tests/worker/jobs/external_services/test_clingen.py +++ b/tests/worker/jobs/external_services/test_clingen.py @@ -917,6 +917,171 @@ async def test_submit_score_set_mappings_to_car_force_reregister_caid_conflict( assert event.event_metadata["clingen_allele_id"] == "CA_STORED" assert event.event_metadata["conflicting_caid"] == "CA_DIFFERENT" + async def _add_derived_alleles_with_distinct_hgvs(self, session, count): + """Attach ``count`` extra alleles to the score set, each carrying a distinct HGVS so the CAR + job produces ``count + 1`` distinct submission lines (one per HGVS). Returns every allele.""" + authoritative_allele = session.scalars(select(Allele)).one() + a_link = session.scalars( + select(MappingRecordAllele).where(MappingRecordAllele.allele_id == authoritative_allele.id) + ).first() + + alleles = [authoritative_allele] + for i in range(count): + post_mapped = deepcopy(authoritative_allele.post_mapped) + post_mapped["expressions"][0]["value"] = f"NC_000001.11:g.{1000 + i}A>T" + derived = Allele( + vrs_digest=f"{authoritative_allele.vrs_digest}-d{i}", + level=authoritative_allele.level, + post_mapped=post_mapped, + ) + session.add(derived) + session.flush() + session.add( + MappingRecordAllele( + mapping_record_id=a_link.mapping_record_id, + allele_id=derived.id, + is_authoritative=False, + ) + ) + alleles.append(derived) + + session.flush() + return alleles + + async def test_submit_score_set_mappings_to_car_batches_submissions( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A large allele set is dispatched in fixed-size chunks rather than one PUT: 4 distinct HGVS + with a batch size of 2 yields two dispatch calls, and every allele is still registered.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + alleles = await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + dispatched_batches = [] + + def fake_dispatch(hgvs_list): + dispatched_batches.append(list(hgvs_list)) + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + + # 4 distinct HGVS at batch size 2 → two dispatch calls, neither exceeding the batch size, and + # together covering every submitted line exactly once. + assert len(dispatched_batches) == 2 + assert all(len(batch) <= 2 for batch in dispatched_batches) + assert sum(len(batch) for batch in dispatched_batches) == 4 + + # Every allele across both batches is registered. + for allele in alleles: + session.refresh(allele) + assert allele.clingen_allele_id is not None + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert all(event.disposition == "present" for event in events) + + async def test_submit_score_set_mappings_to_car_batch_failure_isolated( + self, + mock_worker_ctx, + session, + with_submit_score_set_mappings_to_car_job, + submit_score_set_mappings_to_car_sample_job_run, + mock_s3_client, + sample_score_dataframe, + sample_count_dataframe, + with_dummy_setup_jobs, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ): + """A failed batch fails only its own alleles: the first chunk returns nothing (request failure) + while the second registers normally. Per-batch reconciliation keeps the two independent.""" + await create_mappings_in_score_set( + session, + mock_s3_client, + mock_worker_ctx, + sample_score_dataframe, + sample_count_dataframe, + dummy_variant_creation_job_run, + dummy_variant_mapping_job_run, + ) + await self._add_derived_alleles_with_distinct_hgvs(session, count=3) + + call_count = {"n": 0} + + def fake_dispatch(hgvs_list): + call_count["n"] += 1 + # First batch mimics a request failure (dispatch_submissions returns [] on error); the rest + # register normally. + if call_count["n"] == 1: + return [] + return [{"@id": f"CA/{hgvs}", "type": "nucleotide", "genomicAlleles": []} for hgvs in hgvs_list] + + with ( + patch( + "mavedb.worker.jobs.external_services.clingen.ClinGenAlleleRegistryService.dispatch_submissions", + side_effect=fake_dispatch, + ), + patch("mavedb.worker.jobs.external_services.clingen.DEFAULT_CAR_SUBMISSION_BATCH_SIZE", 2), + patch("mavedb.worker.jobs.external_services.clingen.CAR_SUBMISSION_ENDPOINT", "http://fake-endpoint"), + patch("mavedb.worker.jobs.external_services.clingen.CLIN_GEN_SUBMISSION_ENABLED", True), + ): + result = await submit_score_set_mappings_to_car( + mock_worker_ctx, + submit_score_set_mappings_to_car_sample_job_run.id, + JobManager(session, mock_worker_ctx["redis"], submit_score_set_mappings_to_car_sample_job_run.id), + ) + + # The good batch links alleles, so the job overall succeeds despite the failed batch. + assert result.status == JobStatus.SUCCEEDED + assert call_count["n"] == 2 + + # Exactly the second batch's two alleles are registered; the first batch's two are not. + registered = session.scalars(select(Allele).where(Allele.clingen_allele_id.isnot(None))).all() + assert len(registered) == 2 + + events = session.scalars( + select(AnnotationEvent).where(AnnotationEvent.annotation_type == "clingen_allele_id") + ).all() + assert len(events) == 4 + assert sum(event.disposition == "present" for event in events) == 2 + failed = [event for event in events if event.disposition == "failed"] + assert len(failed) == 2 + assert all(event.reason == "api_error" for event in failed) + @pytest.mark.integration @pytest.mark.asyncio From 3b9f7da6c5d54aa320f934ab833e59e57e6a6d08 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 15 Jul 2026 11:22:33 -0700 Subject: [PATCH 75/93] feat(worker): stop linking protein-level alleles to ClinVar and gnomAD ClinVar and gnomAD are nucleotide-level resources keyed on genomic/coding changes. A protein allele's clinical calls and population frequencies are carried by its underlying g./c. siblings, so linking them at the protein level is a category error now that projection pairing exposes those siblings. Exclude protein-level alleles from both linking jobs. - Carry the allele's sequence level on ScoreSetAlleleRow so nucleotide-only jobs can filter without a second query - Add EventReason.PROTEIN_LEVEL_ALLELE and record protein alleles as a not-applicable structural gap (mirrors the multi-variant-CAID skip) - refresh_clinvar_controls: skip protein alleles first in the loop, before any ClinGen resolution - link_gnomad_variants: drop protein alleles from the Athena query set and skip them in the annotation loop, sparing the round-trip --- src/mavedb/lib/clingen/alleles.py | 10 ++- src/mavedb/models/enums/event_reason.py | 3 + .../worker/jobs/external_services/clinvar.py | 63 ++++++++++++------- .../worker/jobs/external_services/gnomad.py | 37 ++++++++--- .../jobs/external_services/test_clinvar.py | 44 +++++++++++++ .../jobs/external_services/test_gnomad.py | 41 ++++++++++++ 6 files changed, 165 insertions(+), 33 deletions(-) diff --git a/src/mavedb/lib/clingen/alleles.py b/src/mavedb/lib/clingen/alleles.py index 58cb6d7d0..562dab94e 100644 --- a/src/mavedb/lib/clingen/alleles.py +++ b/src/mavedb/lib/clingen/alleles.py @@ -24,8 +24,10 @@ class ScoreSetAlleleRow(NamedTuple): per allele. ``hgvs_g``/``hgvs_c``/``hgvs_p`` are allele-level (stable by construction), carried here so the - VEP job can build its HGVS payload without a second query. They are optional with a ``None`` - default so payloads keying only on the CAID (gnomAD/ClinVar) need not name them. + VEP job can build its HGVS payload without a second query. ``level`` is the allele's sequence + level (``protein``/``cdna``/``genomic``), carried so jobs annotating just nucleotide alleles can + skip protein alleles without a second query. All are optional with a ``None`` default so payloads keying + only on the CAID need not name them; real DB rows always populate them. """ allele_id: int @@ -35,6 +37,7 @@ class ScoreSetAlleleRow(NamedTuple): hgvs_g: str | None = None hgvs_c: str | None = None hgvs_p: str | None = None + level: str | None = None def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAlleleRow]: @@ -56,6 +59,7 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl Allele.hgvs_g, Allele.hgvs_c, Allele.hgvs_p, + Allele.level, ) .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) @@ -67,7 +71,7 @@ def get_alleles_for_score_set(db: Session, score_set_id: int) -> list[ScoreSetAl ).all() return [ - ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p) + ScoreSetAlleleRow(r.id, r.post_mapped, r.clingen_allele_id, r.variant_id, r.hgvs_g, r.hgvs_c, r.hgvs_p, r.level) for r in rows ] diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index f7fd4b56b..35aa23af4 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -30,6 +30,9 @@ class EventReason(str, Enum): NO_CAID = "no_caid" # no ClinGen allele id to key on (gnomAD, ClinVar) NO_HGVS = "no_hgvs" # no HGVS to submit/resolve (CAR, VEP) MULTI_VARIANT_CAID = "multi_variant_caid" # cis-block CAID cannot be used (ClinVar) + PROTEIN_LEVEL_ALLELE = ( + "protein_level_allele" # protein allele excluded from nucleotide-level annotation (ClinVar, gnomAD) + ) NO_ASSAY_LEVEL_HGVS = "no_assay_level_hgvs" # no assay-level HGVS to translate (RT) # failed — errored diff --git a/src/mavedb/worker/jobs/external_services/clinvar.py b/src/mavedb/worker/jobs/external_services/clinvar.py index c3fcf2f2c..557919299 100644 --- a/src/mavedb/worker/jobs/external_services/clinvar.py +++ b/src/mavedb/worker/jobs/external_services/clinvar.py @@ -30,6 +30,7 @@ from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import FailureCategory +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.score_set import ScoreSet from mavedb.worker.jobs.utils.setup import validate_job_params from mavedb.worker.lib.decorators.pipeline_management import with_pipeline_management @@ -120,34 +121,38 @@ async def refresh_clinvar_controls(ctx: dict, job_id: int, job_manager: JobManag versions = _generate_clinvar_versions() - job_manager.save_to_context({ - "application": "mavedb-worker", - "function": "refresh_clinvar_controls", - "resource": score_set.urn, - "correlation_id": correlation_id, - "versions": versions, - "total_versions": len(versions), - "force": force, - }) + job_manager.save_to_context( + { + "application": "mavedb-worker", + "function": "refresh_clinvar_controls", + "resource": score_set.urn, + "correlation_id": correlation_id, + "versions": versions, + "total_versions": len(versions), + "force": force, + } + ) job_manager.update_progress(0, 100, f"Starting ClinVar refresh across {len(versions)} versions.") logger.info(f"Starting ClinVar refresh across {len(versions)} versions", extra=job_manager.logging_context()) - # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys on is often - # the RT-derived one. Events are allele-keyed, so every allele is recorded per release (no fan-out). - allele_data = group_alleles_for_annotation( - get_alleles_for_score_set(job_manager.db, score_set.id), - payload=lambda row: row.clingen_allele_id, - ) + # One work-unit per allele (payload = CAID; alleles without one are dropped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele ClinVar keys + # on is often the RT-derived one. Events are allele-keyed, so every allele is recorded per release + # (no fan-out). + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} job_manager.save_to_context({"num_alleles_with_caids": len(allele_data)}) # Link counts accumulate across all versions (an allele may link in every release it appears in). - annotation_counts: Counter[str] = Counter({ - "created_link_count": 0, - "preexisting_link_count": 0, - "skipped_link_count": 0, - "failed_link_count": 0, - }) + annotation_counts: Counter[str] = Counter( + { + "created_link_count": 0, + "preexisting_link_count": 0, + "skipped_link_count": 0, + "failed_link_count": 0, + } + ) if not allele_data: logger.warning( @@ -204,6 +209,20 @@ def alleles_linked_at_version(clinvar_version: str) -> set[int]: job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) for allele_id, caid in allele_data.items(): + # ClinVar is a nucleotide-level database, skip any protein level alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_link_count"] += 1 + _annotate_clinvar( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + source_version=clinvar_version, + error_message="Protein-level alleles are not linked to ClinVar; see nucleotide siblings.", + metadata={"clingen_allele_id": caid}, + ) + continue + if allele_id in already_linked: annotation_counts["preexisting_link_count"] += 1 _annotate_clinvar( diff --git a/src/mavedb/worker/jobs/external_services/gnomad.py b/src/mavedb/worker/jobs/external_services/gnomad.py index 3b7a48d16..31e541568 100644 --- a/src/mavedb/worker/jobs/external_services/gnomad.py +++ b/src/mavedb/worker/jobs/external_services/gnomad.py @@ -24,6 +24,7 @@ from mavedb.models.enums.annotation_type import AnnotationType from mavedb.models.enums.disposition import Disposition from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.score_set import ScoreSet @@ -111,13 +112,12 @@ async def link_gnomad_variants(ctx: dict, job_id: int, job_manager: JobManager) job_manager.update_progress(0, 100, "Starting gnomAD mapped resource linkage.") logger.info(msg="Started gnomAD mapped resource linkage", extra=job_manager.logging_context()) - # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers ALL the score - # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows is often the - # RT-derived one. Events are allele-keyed, so every linked allele is recorded. - allele_data = group_alleles_for_annotation( - get_alleles_for_score_set(job_manager.db, score_set.id), - payload=lambda row: row.clingen_allele_id, - ) + # One work-unit per allele (payload = CAID; alleles without one are skipped). Covers the score + # set's alleles — authoritative and RT-derived — since the genomic allele gnomAD knows + # is often the RT-derived one. Events are allele-keyed, so every linked allele is recorded. + allele_rows = get_alleles_for_score_set(job_manager.db, score_set.id) + allele_data = group_alleles_for_annotation(allele_rows, payload=lambda row: row.clingen_allele_id) + allele_levels = {row.allele_id: row.level for row in allele_rows} annotation_counts: Counter[str] = Counter( { @@ -159,7 +159,15 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: # Skip alleles already linked at the current version (they can't change). force re-fetches # all; the linker still supersedes only on change, so a forced no-op writes nothing. already_current = set() if force else alleles_linked_at_current_version(set(allele_data.keys())) - variant_caids = sorted({allele_data[aid] for aid in allele_data if aid not in already_current}) + # Never query Athena for protein alleles — gnomAD is nucleotide-level, so they can't match. They + # are still recorded as not-applicable in the loop below; this only spares them the round-trip. + variant_caids = sorted( + { + allele_data[aid] + for aid in allele_data + if aid not in already_current and allele_levels.get(aid) != SequenceLevel.protein.value + } + ) job_manager.save_to_context( { @@ -195,6 +203,19 @@ def alleles_linked_at_current_version(allele_ids: set[int]) -> set[int]: job_manager.db, job_run_id=job_manager.job_id, score_set_id=score_set.id ) for allele_id, caid in allele_data.items(): + # gnomAD is a nucleotide-level, genomic-coordinate resource. Skip any protein alleles. + if allele_levels.get(allele_id) == SequenceLevel.protein.value: + annotation_counts["skipped_allele_count"] += 1 + _annotate_gnomad( + annotation_manager, + allele_id, + Disposition.NOT_APPLICABLE, + EventReason.PROTEIN_LEVEL_ALLELE, + error_message="Protein-level alleles are not linked to gnomAD; see nucleotide siblings.", + metadata={"clingen_allele_id": caid}, + ) + continue + verdict = verdicts.get(allele_id) if verdict is GnomadLinkVerdict.CREATED: annotation_counts["created_allele_count"] += 1 diff --git a/tests/worker/jobs/external_services/test_clinvar.py b/tests/worker/jobs/external_services/test_clinvar.py index 47be57383..f7698d737 100644 --- a/tests/worker/jobs/external_services/test_clinvar.py +++ b/tests/worker/jobs/external_services/test_clinvar.py @@ -132,6 +132,50 @@ async def test_multi_variant_caid_skipped( assert event.reason == EventReason.MULTI_VARIANT_CAID assert event.allele_id == allele.id and event.variant_id is None + async def test_protein_level_allele_skipped( + self, + mock_worker_ctx, + session, + with_refresh_clinvar_controls_job, + sample_refresh_clinvar_controls_job_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to ClinVar (a nucleotide-level DB): its clinical + calls come from its g./c. siblings. Skipped as not-applicable before any resolution runs.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with ( + patch( + "mavedb.worker.jobs.external_services.clinvar.get_associated_clinvar_allele_id", + return_value="VCV000000123", + ) as resolve_spy, + patch( + "mavedb.worker.jobs.external_services.clinvar.fetch_clinvar_variant_data", + return_value=MOCK_CLINVAR_DATA, + ), + ): + result = await refresh_clinvar_controls( + mock_worker_ctx, + sample_refresh_clinvar_controls_job_run.id, + JobManager(session, mock_worker_ctx["redis"], sample_refresh_clinvar_controls_job_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(ClinvarAlleleLink)).all() == [] + assert session.scalars(select(ClinvarControl)).all() == [] + # Short-circuited before touching ClinGen — a protein allele can never resolve to a ClinVar id. + resolve_spy.assert_not_awaited() + + # Allele-keyed not-applicable event: ClinVar is nucleotide-level, so a protein allele is a + # structural gap (like a multi-variant CAID), not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.CLINVAR_CONTROL + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_no_associated_clinvar_allele_id_skipped( self, mock_worker_ctx, diff --git a/tests/worker/jobs/external_services/test_gnomad.py b/tests/worker/jobs/external_services/test_gnomad.py index 0b14fda21..03a3cefac 100644 --- a/tests/worker/jobs/external_services/test_gnomad.py +++ b/tests/worker/jobs/external_services/test_gnomad.py @@ -11,6 +11,9 @@ from mavedb.lib import gnomad as gnomad_lib from mavedb.lib.gnomad import GNOMAD_DATA_VERSION from mavedb.lib.types.workflow import JobExecutionOutcome +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant @@ -72,6 +75,44 @@ async def test_link_gnomad_variants_no_gnomad_matches( assert isinstance(result, JobExecutionOutcome) assert result.status == JobStatus.SUCCEEDED + async def test_link_gnomad_variants_protein_level_allele_skipped( + self, + session, + with_populated_domain_data, + with_gnomad_linking_job, + mock_worker_ctx, + sample_link_gnomad_variants_run, + setup_sample_alleles_with_caid, + ): + """A protein-level allele is never linked to gnomAD (a nucleotide-level, genomic-coordinate + resource): its frequencies come from its g./c. siblings. Skipped as not-applicable, and it + never reaches the Athena query — sparing the round-trip.""" + _, allele = setup_sample_alleles_with_caid + allele.level = "protein" + session.commit() + + with patch( + "mavedb.worker.jobs.external_services.gnomad.gnomad_variant_data_for_caids", + ) as fetch_spy: + result = await link_gnomad_variants( + mock_worker_ctx, + 1, + JobManager(session, mock_worker_ctx["redis"], sample_link_gnomad_variants_run.id), + ) + + assert result.status == JobStatus.SUCCEEDED + assert session.scalars(select(GnomadAlleleLink)).all() == [] + # The protein allele's CAID was dropped from the query set, so Athena is never queried. + fetch_spy.assert_not_called() + + # Allele-keyed not-applicable event: gnomAD is nucleotide-level, so a protein allele is a + # structural gap, not a statement about the source. + event = session.scalars(select(AnnotationEvent)).one() + assert event.annotation_type == AnnotationType.GNOMAD_ALLELE_FREQUENCY + assert event.disposition == Disposition.NOT_APPLICABLE + assert event.reason == EventReason.PROTEIN_LEVEL_ALLELE + assert event.allele_id == allele.id and event.variant_id is None + async def test_link_gnomad_variants_call_linking_method( self, session, From 5751e9c2c455200ae68582d36a791d3f4042231c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 09:22:35 -0700 Subject: [PATCH 76/93] feat(cat-vrs): add co_encodes relation and convergent derivation Under a nucleotide assay the reverse-translation fan leaves synonymous "cousins" on the record: nt alleles in other projection groups that encode the same protein consequence as the measured change. They are distinct, unmeasured variants, not representations of the measured change, and were previously mislabelled as candidates/projections. - Cat-VRS: add the CO_ENCODES relation and carry cousins as members in the full-closure detail object (include_convergent=True); the VA-Spec subject (include_convergent=False) drops them, keeping only the measured change's coordinate partner and protein consequence. - variant detail: add the `convergent` derivation, held in lockstep with the co_encodes relation, so cousins are no longer labelled `candidate` (which now means only protein-assay reverse-translation ambiguity). - relation/derivation now key off projection_group to distinguish the measured change's coordinate partner (same group) from a cousin (a different group). --- src/mavedb/lib/cat_vrs.py | 107 ++++++++++++++++++++--- src/mavedb/lib/variant_detail.py | 66 +++++++++----- src/mavedb/view_models/variant_detail.py | 5 +- tests/lib/test_cat_vrs.py | 101 +++++++++++++++++++-- tests/lib/test_variant_detail.py | 22 ++++- 5 files changed, 254 insertions(+), 47 deletions(-) diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index 10f1920ec..b9ad6d39c 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -28,29 +28,32 @@ from sqlalchemy.orm import Session from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.annotation.util import vrs_object_from_mapped_variant from mavedb.lib.logging.context import logging_context +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) -# MaveDB-namespaced relation codes, read as `member -> defining` (the star model, design D54). No -# off-the-shelf vocabulary fits, so these are namespaced now and raised with the GA4GH Cat-VRS group -# for standardization; `mappings` to a standard term can be added later without a shape change. +# MaveDB-namespaced relation codes, read as `member -> defining`. Namespaced for MaveDB now, +# TODO: Harmonize with GA4GH Cat-VRS group for standardization. _RELATION_SYSTEM = "https://mavedb.org/cat-vrs/relations" +# TODO: Verify Cat-VRS relation names with GA4GH Cat-VRS group; these are the MaveDB-namespaced codes we raised for standardization. class CatVrsRelation(str, Enum): """Relation of a member allele to the single defining (measured) allele.""" - # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. - COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is protein; member is an nt allele (coding or genomic) that encodes it. Implied. ENCODES = "encodes" + # defining is nt; member is the same variant in other nt coordinates (genomic<->coding). Faithful. + COORDINATE_REPRESENTATION_OF = "coordinate_representation_of" # defining is nt; member is the protein consequence. Consequence, no independent score. TRANSLATION_OF = "translation_of" + # defining is nt; member is a *synonymous cousin* — an nt allele in a different projection group that + # encodes the same protein consequence as the measured change. + CO_ENCODES = "co_encodes" class CatVrsMode(str, Enum): @@ -73,25 +76,67 @@ class CategoricalVariantTransit: member_relations: dict[str, CatVrsRelation] -def _relation_for(defining_level: Optional[str], member_level: Optional[str]) -> Optional[CatVrsRelation]: +def is_convergent_cousin( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """Whether a projection-mode member is a *synonymous cousin* of the measured change. + + Concretely: + - The defining (measured) allele is nt (genomic or cdna). + - The member is nt (genomic or cdna). + - The member is in a different projection group than the defining (measured) allele. + """ + return ( + member_level in _NUCLEOTIDE_LEVELS + and member_group is not None + and defining_group is not None + and member_group != defining_group + ) + + +def _relation_for( + defining_level: Optional[str], + member_level: Optional[str], + *, + defining_group: Optional[int] = None, + member_group: Optional[int] = None, +) -> Optional[CatVrsRelation]: """Derive the member -> defining relation from the two levels, or None for the defining itself. - Star model: every member relates to the single defining allele, so the relation depends only on - the (defining, member) level pair, never on a chain between members. + Star model: every member relates to the single defining allele, so the relation depends only on the + (defining, member) level pair — plus, in projection mode, on whether the nt member shares the measured + change's ``projection_group`` (its coordinate partner) or lives in another group (a synonymous cousin). """ - # Mode 2 — protein measured. Every nt member encodes it; the protein member is the defining. + # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. if defining_level == SequenceLevel.protein.value: return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None - # Mode 1 — nt measured (genomic or cdna). + # Nt measured (genomic or cdna). if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF if member_level in _NUCLEOTIDE_LEVELS: + if is_convergent_cousin(member_level, member_group, defining_group=defining_group): + return CatVrsRelation.CO_ENCODES return CatVrsRelation.COORDINATE_REPRESENTATION_OF return None # pragma: no cover -- the levels are constrained by the DB and the mapping job +def _is_precise_projection_member( + member_level: Optional[str], member_group: Optional[int], *, defining_group: Optional[int] +) -> bool: + """In projection mode, whether a member is a *precise* representation of the measured change. + + True for the protein consequence (the apex, shared by the whole equivalence class) and the measured + change's own coordinate partner — the other member of its ``projection_group`` (a c↔g pair). False for + the synonymous cousins in other projection groups. + """ + if member_level == SequenceLevel.protein.value: + return True + + return not is_convergent_cousin(member_level, member_group, defining_group=defining_group) + + def _relation_concept(relation: CatVrsRelation) -> MappableConcept: """Wrap a MaveDB relation code as a Cat-VRS ``MappableConcept`` for the constraint.""" return MappableConcept( @@ -115,12 +160,30 @@ def _hydrate_vrs(allele: Allele) -> Optional[VrsAllele | CisPhasedBlock]: return variation -def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> Optional[CategoricalVariantTransit]: +def build_categorical_variant( + links: list[MappingRecordAllele], *, name: str, include_convergent: bool = True +) -> Optional[CategoricalVariantTransit]: """Assemble a Cat-VRS ``CategoricalVariant`` from a variant's live allele links. ``links`` is the record-scoped live link set from ``get_live_record_allele_links`` — exactly one is authoritative (the measured/defining allele), the rest are derived members. Returns ``None`` when there is no authoritative, hydratable link to anchor on (e.g. an unmapped variant). + + **The categorical variant always anchors on the measured allele**. Member selection therefore differs by mode: + + - **Reverse translation** (protein measured): the defining allele is the protein change and every + nt allele in the record ``encodes`` it, so the **full equivalence class** is unfurled — that class + is exactly what the protein measurement's claim ranges over. ``include_convergent`` is inert here. + - **Projection** (nt measured): the record also carries the reverse-translation fan — the synonymous + *cousins* (other nt encoders of the same protein consequence, in different projection groups), which + are distinct, unmeasured variants rather than representations of the measured change. + ``include_convergent`` selects how they are handled: + + - ``True`` (default; the detail envelope): the cousins are kept as members wearing the + :attr:`CatVrsRelation.CO_ENCODES` relation, so the object is the full closure. The measured change's + coordinate partner and protein consequence stay ``coordinate_representation_of`` / ``translation_of``. + - ``False`` (the VA-Spec subject): the cousins are dropped (see :func:`_is_precise_projection_member`) + and only the measured change's precise coordinate partner and protein consequence remain. """ defining_link = next((link for link in links if link.is_authoritative), None) if defining_link is None: @@ -152,6 +215,19 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> continue allele = link.allele + + # Projection mode, narrow object (include_convergent=False): drop any synonymous cousins the + # reverse-translation fan left on the record, keeping only the measured change's precise coordinate + # partner and its protein consequence. + if ( + mode is CatVrsMode.PROJECTION + and not include_convergent + and not _is_precise_projection_member( + allele.level, link.projection_group, defining_group=defining_link.projection_group + ) + ): + continue + member_vrs = _hydrate_vrs(allele) # A live link to an allele with no post_mapped is an unexpected data state. if member_vrs is None: @@ -165,7 +241,12 @@ def build_categorical_variant(links: list[MappingRecordAllele], *, name: str) -> continue members.append(member_vrs) - relation = _relation_for(defining_level, allele.level) + relation = _relation_for( + defining_level, + allele.level, + defining_group=defining_link.projection_group, + member_group=link.projection_group, + ) if relation is not None and allele.vrs_digest is not None: member_relations[allele.vrs_digest] = relation relations_present[relation] = None diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 987e0e3ea..8f06c7cc5 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -26,7 +26,7 @@ from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations from mavedb.lib.alleles import get_live_record_allele_links -from mavedb.lib.cat_vrs import categorical_variant_for_variant +from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_cousin from mavedb.lib.score_calibrations import calibration_preference_key from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord @@ -65,12 +65,18 @@ class AlleleDerivation(str, Enum): # The assay's actual measurement (the authoritative link). Precise by definition. AUTHORITATIVE = "authoritative" - # Deterministic and precise. Every allele derived from a *nucleotide* measurement is a projection: - # nucleotide↔nucleotide and nucleotide→protein are both deterministic given (assembly, transcript). + # Deterministic and precise. Derived from the measured change itself: nucleotide↔nucleotide (its + # coordinate partner) and nucleotide→protein (its consequence) are both deterministic given + # (assembly, transcript). PROJECTION = "projection" # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous # codons). One member of the fanned-out equivalence class, not a precise coordinate. CANDIDATE = "candidate" + # A distinct, precisely-known nucleotide change that *converges* on the measured protein consequence + # from a different codon (a synonymous cousin under a nucleotide assay). Not ambiguous like a + # candidate, and not a projection *of* the measured change — a separate, unmeasured variant that + # merely shares the consequence. Pairs with the ``co_encodes`` Cat-VRS relation. + CONVERGENT = "convergent" @dataclass(frozen=True) @@ -87,10 +93,11 @@ class AlleleIdentity: (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured allele, or when the allele is not a Cat-VRS member. Sourced from ``cat_vrs.CategoricalVariantTransit.member_relations``. - - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / - candidate. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide - assay has ``relation=translation_of`` but ``derivation=projection``, whereas the same-shaped - member of a protein assay is ``derivation=candidate``). + - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / candidate / + convergent. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide + assay has ``relation=translation_of`` but ``derivation=projection``; a synonymous cousin has + ``relation=co_encodes`` and ``derivation=convergent``; the nucleotide fan-out of a protein assay is + ``derivation=candidate``). - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), for pre-reverse-translation data, and where a level's projection failed. @@ -170,26 +177,34 @@ def _classifications_for_variant( ] -def _derivation_for(*, is_authoritative: bool, assay_level: Optional[SequenceLevel]) -> AlleleDerivation: +def _derivation_for( + *, is_authoritative: bool, assay_level: Optional[SequenceLevel], is_cousin: bool +) -> AlleleDerivation: """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. - The measured allele is ``authoritative``. Every *other* allele's confidence is set by the *only* - ambiguous boundary in the stack — protein → codon: - - - **nucleotide assay** (``assay_level`` genomic/cdna) → ``projection`` for all derived alleles. - nucleotide↔nucleotide and nucleotide→protein are deterministic, so the whole equivalence class - is precise. - - **protein assay** (``assay_level`` protein) → ``candidate`` for the derived (nucleotide) fan-out. - Reverse translation is genuinely ambiguous. - - Only ``assay_level`` (not the allele's own level) distinguishes the two, because on a nucleotide - assay every derived level is deterministic and on a protein assay every derived allele is part of - the ambiguous nucleotide fan-out. + The measured allele is ``authoritative``. Every *other* allele's provenance turns on the protein → + codon boundary, which surfaces three ways: + + - **protein assay** (``assay_level`` protein) → ``candidate`` for the whole derived (nucleotide) + fan-out: reverse translation is genuinely ambiguous (which codon was measured is unknown). + - **nucleotide assay** (``assay_level`` genomic/cdna), synonymous *cousin* (``is_cousin``) → an nt + member in a different projection group: a distinct, precisely-known variant that merely + *converges* on the measured protein consequence. It is ``convergent`` — not ambiguous (so not a + ``candidate``) and not a projection *of* the measured change. + - **nucleotide assay**, otherwise → ``projection`` for the precise class derived from the measured + change itself (its coordinate partner and its protein consequence, both deterministic). + + ``is_cousin`` is :func:`cat_vrs.is_convergent_cousin` for this allele, keeping the ``derivation`` + (provenance) axis in lockstep with the Cat-VRS ``relation`` axis — ``co_encodes`` ↔ ``convergent``, + ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, protein-assay ``encodes`` ↔ + ``candidate``. """ if is_authoritative: return AlleleDerivation.AUTHORITATIVE if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE + if is_cousin: + return AlleleDerivation.CONVERGENT return AlleleDerivation.PROJECTION @@ -263,11 +278,13 @@ def get_variant_detail( if link.projection_group is not None and link.allele.vrs_digest is not None: group_digests.setdefault(link.projection_group, []).append(link.allele.vrs_digest) + # The measured change's projection group anchors the cousin test: an nt member in a *different* group + # is a synonymous cousin (co_encodes / convergent), not the measured change's coordinate partner. + defining_group = authoritative.projection_group if authoritative is not None else None + alleles: dict[str, AlleleIdentity] = {} for link in links: allele = link.allele - if allele.vrs_digest is None: - continue # The projection sibling: the other digest in this link's projection_group, if any. A group has # ≤2 members, so there is at most one sibling — this allele's projection partner. @@ -279,12 +296,15 @@ def get_variant_detail( ) relation = relations.get(allele.vrs_digest) + is_cousin = is_convergent_cousin(allele.level, link.projection_group, defining_group=defining_group) alleles[allele.vrs_digest] = AlleleIdentity( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, relation=relation.value if relation is not None else None, - derivation=_derivation_for(is_authoritative=link.is_authoritative, assay_level=assay_level).value, + derivation=_derivation_for( + is_authoritative=link.is_authoritative, assay_level=assay_level, is_cousin=is_cousin + ).value, projection_of=projection_of, ) diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 190f41d7a..31f5518aa 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -40,8 +40,9 @@ class AlleleIdentity(BaseModel): - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured allele, or when the allele is not a Cat-VRS member. - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, - precise) / ``candidate`` (reverse-translation, ambiguous). Orthogonal to ``relation`` — never - conflate them. + precise, derived from the measured change) / ``candidate`` (protein-assay reverse-translation, + ambiguous) / ``convergent`` (a distinct, precise nucleotide change that converges on the measured + protein consequence). Orthogonal to ``relation``. Do not conflate them. - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. """ diff --git a/tests/lib/test_cat_vrs.py b/tests/lib/test_cat_vrs.py index ba6f54e47..47eb6ac64 100644 --- a/tests/lib/test_cat_vrs.py +++ b/tests/lib/test_cat_vrs.py @@ -57,16 +57,20 @@ def _post_mapped() -> dict: _DEFAULT_POST_MAPPED = object() -def _link(*, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED) -> MappingRecordAllele: +def _link( + *, level: str, digest: str, is_authoritative: bool, post_mapped=_DEFAULT_POST_MAPPED, projection_group=None +) -> MappingRecordAllele: """A transient (record, allele) link with its allele attached — no session needed. ``digest`` is the ``Allele.vrs_digest`` *column* (the key the builder uses for member relations), deliberately distinct from the spec-valid VRS digest embedded in post_mapped. Pass - ``post_mapped=None`` to model an un-hydratable allele. + ``post_mapped=None`` to model an un-hydratable allele. ``projection_group`` pairs a c↔g projection + (the two links of one precise change share a value; the protein apex carries ``None``); the builder + uses it in projection mode to keep only the measured change's precise coordinate partner. """ pm = _post_mapped() if post_mapped is _DEFAULT_POST_MAPPED else post_mapped allele = Allele(level=level, vrs_digest=digest, post_mapped=pm) - return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele) + return MappingRecordAllele(is_authoritative=is_authoritative, allele=allele, projection_group=projection_group) @pytest.mark.unit @@ -101,10 +105,10 @@ def test_mode_2_protein_measured_reverse_translation(): @pytest.mark.unit def test_mode_1_coding_measured_projection(): - """Coding measured: nt sibling is a coordinate representation; protein member is a translation.""" + """Coding measured: the projection_group partner is a coordinate representation; protein is a translation.""" links = [ - _link(level="cdna", digest="cdna", is_authoritative=True), - _link(level="genomic", digest="gen", is_authoritative=False), + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), _link(level="protein", digest="prot", is_authoritative=False), ] @@ -122,6 +126,91 @@ def test_mode_1_coding_measured_projection(): assert codes == {"coordinate_representation_of", "translation_of"} +@pytest.mark.unit +def test_mode_1_projection_includes_sibling_encoders(): + """Coding measured, full closure (default include_convergent=True): the reverse-translation fan on the + record (other encoders of the same protein consequence, in *different* projection groups) is kept as + members wearing the ``co_encodes`` relation, so the object is the full closure. The measured change's + own coordinate partner and protein consequence stay coordinate_representation_of / translation_of.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + # A sibling encoder of the same protein change — a distinct variant, different projection group. + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b") + assert transit is not None + + # The coordinate partner + protein consequence keep their faithful relations; the two cousins ride as + # co_encodes (distinct, unmeasured synonymous variants). + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + "sibling_cdna": CatVrsRelation.CO_ENCODES, + "sibling_gen": CatVrsRelation.CO_ENCODES, + } + # members = defining cdna + gen partner + protein apex + the two cousins (full closure). + assert len(transit.categorical_variant.members) == 5 + # The distinct relation kinds surface on the constraint, including co_encodes. + constraint = transit.categorical_variant.constraints[0].root + codes = {str(r.primaryCoding.code.root) for r in constraint.relations} + assert codes == {"coordinate_representation_of", "translation_of", "co_encodes"} + + +@pytest.mark.unit +def test_mode_1_projection_narrow_object_drops_sibling_encoders(): + """Coding measured, narrow object (include_convergent=False, the VA subject): the synonymous cousins + in other projection groups are dropped — only the measured change's coordinate partner and protein + consequence remain. This pins the shape the VA-Spec path builds.""" + links = [ + _link(level="cdna", digest="cdna", is_authoritative=True, projection_group=0), + _link(level="genomic", digest="gen", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="sibling_cdna", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="sibling_gen", is_authoritative=False, projection_group=1), + _link(level="protein", digest="prot", is_authoritative=False), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2b-narrow", include_convergent=False) + assert transit is not None + + # Only the measured change's coordinate partner + the protein consequence; the cousins are excluded. + assert transit.member_relations == { + "gen": CatVrsRelation.COORDINATE_REPRESENTATION_OF, + "prot": CatVrsRelation.TRANSLATION_OF, + } + # members = defining cdna + gen partner + protein apex (the two cousins are dropped). + assert len(transit.categorical_variant.members) == 3 + + +@pytest.mark.unit +def test_mode_2_reverse_translation_keeps_the_full_encoder_class(): + """Protein measured: every nt encoder stays, across projection groups — the full equivalence class + is what the protein claim ranges over (no sibling filtering in reverse-translation mode).""" + links = [ + _link(level="protein", digest="prot", is_authoritative=True), + _link(level="cdna", digest="cdna_a", is_authoritative=False, projection_group=0), + _link(level="genomic", digest="gen_a", is_authoritative=False, projection_group=0), + _link(level="cdna", digest="cdna_b", is_authoritative=False, projection_group=1), + _link(level="genomic", digest="gen_b", is_authoritative=False, projection_group=1), + ] + + transit = build_categorical_variant(links, name="urn:mavedb:test#2c") + assert transit is not None + + assert transit.mode == CatVrsMode.REVERSE_TRANSLATION + # All four nt encoders `encodes` the defining protein; none dropped. + assert transit.member_relations == { + "cdna_a": CatVrsRelation.ENCODES, + "gen_a": CatVrsRelation.ENCODES, + "cdna_b": CatVrsRelation.ENCODES, + "gen_b": CatVrsRelation.ENCODES, + } + assert len(transit.categorical_variant.members) == 5 + + @pytest.mark.unit def test_no_authoritative_link_returns_none(): """An unmapped variant (no authoritative link) has no defining allele to anchor on.""" diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index fa193b53d..4c8a3384c 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -150,11 +150,15 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): measured = _allele(session, "cdna-digest", level="cdna", clingen_allele_id="CA123", hgvs_c="NM_000546.6:c.1216G>A") genomic = _allele(session, "gen-digest", level="genomic", hgvs_g="NC_000017.11:g.7676154C>T") protein = _allele(session, "prot-digest", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr") + # A synonymous cousin: an nt encoder of the same protein consequence in a *different* projection group + # — a distinct, unmeasured variant carried by the reverse-translation fan. + cousin = _allele(session, "cousin-digest", level="cdna", hgvs_c="NM_000546.6:c.1218C>T") # The measured cdna link and its genomic projection share a projection_group (the RT fold-in); the - # protein apex is in no pair (group None). + # protein apex is in no pair (group None); the cousin is in its own group. _link(session, record, measured, is_authoritative=True, projection_group=0) _link(session, record, genomic, projection_group=0) _link(session, record, protein) + _link(session, record, cousin, projection_group=1) _vep(session, measured, "missense_variant") detail = get_variant_detail(session, variant) @@ -169,10 +173,13 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert detail.mode == "projection" assert detail.molecular_representation is not None assert detail.molecular_representation["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar — every linked allele is a + # member, and vice versa (the cousin included). + assert len(detail.molecular_representation["members"]) == len(detail.alleles) # The alleles sidecar carries one identity per linked allele, keyed by digest: level + # reference-frame HGVS (coalesced from hgvs_g/c/p) + ClinGen id + member->defining relation + # derivation + projection_of. - assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest"} + assert set(detail.alleles) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} measured_identity = detail.alleles["cdna-digest"] assert measured_identity.level == "cdna" assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c @@ -195,8 +202,17 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): # projection sibling (the apex is in no c/g pair). assert protein_identity.derivation == "projection" assert protein_identity.projection_of is None + # The synonymous cousin (different projection group) is a member wearing co_encodes (structural), and + # labelled `convergent` (provenance) — a distinct, precise change that converges on the measured + # consequence, not an ambiguous candidate and not a projection of the measured change. It pairs with + # nothing (its group has a single member). + cousin_identity = detail.alleles["cousin-digest"] + assert cousin_identity.level == "cdna" + assert cousin_identity.relation == "co_encodes" + assert cousin_identity.derivation == "convergent" + assert cousin_identity.projection_of is None # Annotations keyed by digest, covering all linked alleles; VEP rode in on the measured allele. - assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest"} + assert set(detail.annotations) == {"cdna-digest", "gen-digest", "prot-digest", "cousin-digest"} assert detail.annotations["cdna-digest"].vep is not None assert detail.annotations["cdna-digest"].vep.consequence == "missense_variant" assert detail.is_current is True From 28f90ea6d292f69c3cb843109d9f5a834f78c883 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 09:51:13 -0700 Subject: [PATCH 77/93] refactor(annotation): dissolve util.py into cohesive modules The annotation util.py had grown into an unrelated grab-bag. Split it so each helper lives with the concept it serves. - Extract VRS rehydration into lib/vrs.py, the annotation eligibility predicates into eligibility.py, calibration selection into calibration.py, and the VariantAnnotationContext builder into context.py. - Co-locate sequence_feature_for_variant in proposition.py, its sole consumer. - Repoint the importers (annotate, classification, document, evidence_line, proposition, statement, study_result) and split the tests to match the new module homes, with patch targets now pointing at the consuming module rather than the old util module. --- src/mavedb/lib/annotation/annotate.py | 105 +-- src/mavedb/lib/annotation/calibration.py | 177 ++++ src/mavedb/lib/annotation/classification.py | 24 +- src/mavedb/lib/annotation/context.py | 80 ++ src/mavedb/lib/annotation/document.py | 31 +- src/mavedb/lib/annotation/eligibility.py | 128 +++ src/mavedb/lib/annotation/evidence_line.py | 18 +- src/mavedb/lib/annotation/proposition.py | 81 +- src/mavedb/lib/annotation/statement.py | 26 +- src/mavedb/lib/annotation/study_result.py | 33 +- src/mavedb/lib/annotation/util.py | 504 ------------ src/mavedb/lib/vrs.py | 134 +++ tests/lib/annotation/conftest.py | 52 ++ tests/lib/annotation/test_annotate.py | 151 ++-- tests/lib/annotation/test_calibration.py | 344 ++++++++ tests/lib/annotation/test_classification.py | 36 +- tests/lib/annotation/test_context.py | 108 +++ tests/lib/annotation/test_document.py | 44 +- tests/lib/annotation/test_eligibility.py | 375 +++++++++ tests/lib/annotation/test_evidence_line.py | 83 +- tests/lib/annotation/test_proposition.py | 133 ++- tests/lib/annotation/test_statement.py | 54 +- tests/lib/annotation/test_study_result.py | 34 +- tests/lib/annotation/test_util.py | 868 -------------------- tests/lib/test_vrs.py | 96 +++ 25 files changed, 1993 insertions(+), 1726 deletions(-) create mode 100644 src/mavedb/lib/annotation/calibration.py create mode 100644 src/mavedb/lib/annotation/context.py create mode 100644 src/mavedb/lib/annotation/eligibility.py delete mode 100644 src/mavedb/lib/annotation/util.py create mode 100644 src/mavedb/lib/vrs.py create mode 100644 tests/lib/annotation/test_calibration.py create mode 100644 tests/lib/annotation/test_context.py create mode 100644 tests/lib/annotation/test_eligibility.py delete mode 100644 tests/lib/annotation/test_util.py create mode 100644 tests/lib/test_vrs.py diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index e5e289a39..e6676f18b 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -1,11 +1,15 @@ """ -This module supports the construction of three main VA-Spec data structures based on the MaveDB MappedVariant object: +This module supports the construction of three main VA-Spec data structures from a variant's +:class:`VariantAnnotationContext` (the new mapping-record substrate — see ``context.py``): - StudyResult See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#study-result-structure - Statement See: https://va-ga4gh.readthedocs.io/en/latest/modeling-foundations/data-structures.html#statement-structure - VariantPathogenicityStatement See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 + +Callers build the context at the edge (router/script) via ``variant_annotation_context`` and pass it in; +these builders never touch the database. ``as_of`` selection and supersession are baked into the context. """ from typing import Optional, Union @@ -13,46 +17,45 @@ from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from mavedb.lib.annotation.classification import functional_classification_of_variant -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line -from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, -) -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, - mapped_variant_to_pathogenicity_statement, -) -from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result -from mavedb.lib.annotation.util import ( - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, +from mavedb.lib.annotation.calibration import ( score_calibration_may_be_used_for_annotation, select_strongest_functional_calibration, select_strongest_pathogenicity_calibration, ) -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.classification import functional_classification_of_variant +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.annotation.eligibility import ( + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) +from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.annotation.proposition import ( + variant_functional_impact_proposition, + variant_pathogenicity_proposition, +) +from mavedb.lib.annotation.statement import functional_statement, pathogenicity_statement +from mavedb.lib.annotation.study_result import variant_impact_study_result -def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: - return mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) +def variant_study_result(context: VariantAnnotationContext) -> ExperimentalVariantFunctionalImpactStudyResult: + return variant_impact_study_result(context) def variant_functional_impact_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + context.variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) # Collect eligible calibrations eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: + for score_calibration in context.variant.score_set.score_calibrations: if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type="functional", @@ -62,7 +65,7 @@ def variant_functional_impact_statement( # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_functional_calibration( - mapped_variant, eligible_calibrations + context.variant, eligible_calibrations ) if not strongest_calibration: @@ -70,33 +73,33 @@ def variant_functional_impact_statement( # Get the classification from the strongest range # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Build evidence lines for all eligible calibrations functional_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) + functional_evidence.append(functional_evidence_line(context, score_calibration, [study_result])) - return mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + return functional_statement( + context, functional_proposition, functional_evidence, strongest_calibration, classification ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + context: VariantAnnotationContext, allow_research_use_only_calibrations: bool = False ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + context.variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations ): return None - study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + study_result = variant_impact_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + clinical_proposition = variant_pathogenicity_proposition(context) # Collect eligible calibrations eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: + for score_calibration in context.variant.score_set.score_calibrations: if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type="pathogenicity", @@ -106,7 +109,7 @@ def variant_pathogenicity_statement( # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( - mapped_variant, eligible_calibrations + context.variant, eligible_calibrations ) if not strongest_calibration: @@ -114,7 +117,7 @@ def variant_pathogenicity_statement( # Get the classification from the strongest range (used for the functional statement within clinical evidence) # If strongest_range is None, the variant is not in any range, so classification will be INDETERMINATE - _, classification = functional_classification_of_variant(mapped_variant, strongest_calibration) + _, classification = functional_classification_of_variant(context.variant, strongest_calibration) # Note: strongest_range is used in the pathogenicity statement for ACMG classification # If None, the statement will use UNCERTAIN_SIGNIFICANCE @@ -122,33 +125,33 @@ def variant_pathogenicity_statement( # Build evidence lines for all eligible calibrations clinical_evidence = [] for score_calibration in eligible_calibrations: - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, [functional_evidence], score_calibration, classification + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + functional_impact_statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, classification ) clinical_evidence.append( - acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + acmg_evidence_line(context, score_calibration, clinical_proposition, [functional_impact_statement]) ) - return mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + return pathogenicity_statement( + context, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range ) def variant_highest_level_annotation( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ - Build the single highest-materialized VA-Spec layer for a mapped variant. + Build the single highest-materialized VA-Spec layer for a variant's annotation context. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. - Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + Returns None when the variant lacks the target/mapping data required to build any layer. """ try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant): - return variant_pathogenicity_statement(mapped_variant) - if can_annotate_variant_for_functional_statement(mapped_variant): - return variant_functional_impact_statement(mapped_variant) - return variant_study_result(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(context.variant): + return variant_pathogenicity_statement(context) + if can_annotate_variant_for_functional_statement(context.variant): + return variant_functional_impact_statement(context) + return variant_study_result(context) except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/calibration.py b/src/mavedb/lib/annotation/calibration.py new file mode 100644 index 000000000..ec6f67f50 --- /dev/null +++ b/src/mavedb/lib/annotation/calibration.py @@ -0,0 +1,177 @@ +"""Score-calibration eligibility and selection for VA-Spec annotation. + +Which of a score set's calibrations may back an annotation (``score_calibration_may_be_used_for_annotation``) +and, among the eligible ones, which carries the strongest evidence for a given variant +(``select_strongest_*``). The selection resolves ties/conflicts to the conservative call (normal / uncertain +significance). Consumed by the annotation builders in ``annotate.py``. +""" + +from typing import Literal, Optional + +from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided + +from mavedb.lib.annotation.classification import ( + ExperimentalVariantFunctionalImpactClassification, + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.variant import Variant + + +def score_calibration_may_be_used_for_annotation( + score_calibration: ScoreCalibration, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a score calibration may be used for annotation based on its properties. + + This function evaluates whether a given score calibration is suitable for use in + annotation by checking its research use only status and the presence of required + classifications based on the annotation type. + + Args: + score_calibration (ScoreCalibration): The score calibration to evaluate. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation + being considered, which determines the required classifications for validity. + allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations + marked as research use only for annotation. Defaults to False. + + Returns: + bool: True if the score calibration can be used for annotation, False otherwise. + """ + if score_calibration.research_use_only and not allow_research_use_only_calibrations: + return False + + if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: + return False + + if annotation_type == "pathogenicity" and all( + fr.acmg_classification is None for fr in score_calibration.functional_classifications + ): + return False + + return True + + +def select_strongest_functional_calibration( + variant: Variant, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for functional classification. + + In case of ties or conflicting classifications, defaults to normal classification. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should be classified as INDETERMINATE but still receive annotations. + """ + if not calibrations: + return None, None + + # Collect all calibrations and their classifications + candidates: list[ + tuple[ + ScoreCalibration, + ScoreCalibrationFunctionalClassification, + ExperimentalVariantFunctionalImpactClassification, + ] + ] = [] + + for calibration in calibrations: + functional_range, classification = functional_classification_of_variant(variant, calibration) + if functional_range is not None: + candidates.append((calibration, functional_range, classification)) + + # If variant is not in any range, return first calibration with None to indicate INDETERMINATE + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Check if all classifications agree + classifications = [c[2] for c in candidates] + if all(cls == classifications[0] for cls in classifications): + # All agree, return the first one + return candidates[0][0], candidates[0][1] + + # Conflict exists: default to normal classification + normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] + if normal_candidates: + return normal_candidates[0][0], normal_candidates[0][1] + + # If no normal classification, return the first candidate + return candidates[0][0], candidates[0][1] + + +def select_strongest_pathogenicity_calibration( + variant: Variant, + calibrations: list[ScoreCalibration], +) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: + """ + Select the calibration with the strongest evidence for pathogenicity classification. + + Uses ACMG evidence strength to determine the strongest evidence. + In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain + significance by returning None for the functional range. + Returns the calibration and its functional classification range that contains the variant. + + If the variant is not contained in any range, returns (first_calibration, None) to indicate + the variant should receive annotations even though it's not classified in any range. + """ + if not calibrations: + return None, None + + # Define evidence strength ordering (higher index = stronger evidence) + # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. + # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. + strength_order = { + None: 0, + VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, + VaSpecStrengthOfEvidenceProvided.MODERATE: 2, + VaSpecStrengthOfEvidenceProvided.STRONG: 3, + VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, + } + + # Collect all calibrations with their evidence strength and criterion + candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] + + for calibration in calibrations: + functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant(variant, calibration) + if functional_range is not None: + strength_value = strength_order.get(evidence_strength, 0) + is_benign = criterion.name.startswith("B") if criterion else False + candidates.append((calibration, functional_range, strength_value, is_benign)) + + # If variant is not in any range, return first calibration with None + if not candidates: + return calibrations[0], None + + # If only one candidate, return it + if len(candidates) == 1: + return candidates[0][0], candidates[0][1] + + # Find the maximum strength + max_strength = max(c[2] for c in candidates) + strongest_candidates = [c for c in candidates if c[2] == max_strength] + + # If only one with max strength, return it + if len(strongest_candidates) == 1: + return strongest_candidates[0][0], strongest_candidates[0][1] + + # Tie: check for conflicting evidence (both benign and pathogenic) + has_benign = any(c[3] for c in strongest_candidates) + has_pathogenic = any(not c[3] for c in strongest_candidates) + + # If there's a conflict between benign and pathogenic evidence of equal strength, + # return None for the functional range to indicate uncertain significance + if has_benign and has_pathogenic: + return strongest_candidates[0][0], None + + # Otherwise return the first of the strongest + return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..9e6e774a6 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -6,7 +6,7 @@ from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -22,22 +22,22 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + variant: Variant, score_calibration: ScoreCalibration ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify functional impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify functional impact." ) @@ -45,7 +45,7 @@ def functional_classification_of_variant( # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if variant in functional_range.variants: if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -56,7 +56,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( - mapped_variant: MappedVariant, + variant: Variant, score_calibration: ScoreCalibration, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], @@ -75,15 +75,15 @@ def pathogenicity_classification_of_variant( Raises ValueError if required calibration, score, or evidence strength is missing. """ - if not mapped_variant.variant.score_set.score_calibrations: + if not variant.score_set.score_calibrations: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have a score set with score calibrations." + f"Variant {variant.urn} does not have a score set with score calibrations." " Unable to classify clinical impact." ) if not score_calibration.functional_classifications: raise ValueError( - f"Variant {mapped_variant.variant.urn} does not have ranges defined in its primary score calibration." + f"Variant {variant.urn} does not have ranges defined in its primary score calibration." " Unable to classify clinical impact." ) @@ -91,7 +91,7 @@ def pathogenicity_classification_of_variant( # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if variant in pathogenicity_range.variants: if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) @@ -130,7 +130,7 @@ def pathogenicity_classification_of_variant( not in VariantPathogenicityEvidenceLine.Criterion._member_names_ ): # pragma: no cover - enforced by model validators in FunctionalClassification view model raise ValueError( - f"Variant {mapped_variant.variant.urn} is contained in a clinical calibration range with an invalid criterion." + f"Variant {variant.urn} is contained in a clinical calibration range with an invalid criterion." " Unable to classify clinical impact." ) diff --git a/src/mavedb/lib/annotation/context.py b/src/mavedb/lib/annotation/context.py new file mode 100644 index 000000000..3e67f6d39 --- /dev/null +++ b/src/mavedb/lib/annotation/context.py @@ -0,0 +1,80 @@ +"""Allele-graph variant context for annotation builders. + +Serves ``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate to VA-Spec builders, +and provides a single point of truth for the live (or as-of) mapping record, authoritative allele, +and pre-built VA proposition subject. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.alleles import get_live_record_allele_links +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.variant import Variant + + +@dataclass +class VariantAnnotationContext: + """A variant's annotation inputs, sourced from it's allele-graph. + + ``record`` is the live (or as-of) ``MappingRecord`` — its ``mapping_api_version`` / ``mapped_date`` + supply VA provenance, and its ``ValidTime`` is what ``as_of`` and supersession are evaluated against. + ``measured_allele`` is the authoritative allele (the assayed representation); its ``post_mapped`` VRS is + the concrete study-result focus. ``subject_variant`` is the VA *proposition* subject. When projections + exist, this is a Cat-VRS ``CategoricalVariant`` object. Otherwise, it is the measured ``MolecularVariation``. + """ + + variant: Variant + record: MappingRecord + measured_allele: Allele + subject_variant: MolecularVariation | CategoricalVariant + as_of: Optional[datetime] + + +def variant_annotation_context( + db: Session, variant: Variant, *, as_of: Optional[datetime] = None +) -> Optional[VariantAnnotationContext]: + """Assemble the annotation context for ``variant`` from the live (or as-of) mapping substrate. + + Returns ``None`` when the variant is unmapped at ``as_of`` — no live mapping record, or an authoritative + allele that carries no ``post_mapped`` VRS (nothing to annotate). One record fetch + one allele-link + fetch; the Cat-VRS transit is built from the same links, so the subject costs no redundant query. + """ + record = db.scalar( + select(MappingRecord).where(MappingRecord.variant_id == variant.id).where(MappingRecord.live_at(as_of)) + ) + if record is None: + return None + + links = get_live_record_allele_links(db, variant.id, as_of=as_of) + measured_allele = next((link.allele for link in links if link.is_authoritative), None) + if measured_allele is None or measured_allele.post_mapped is None: + return None + + # The proposition subject follows the measured as anchor rule: the categorical variant when it carries a + # projection member, else the concrete measured variation. The VA subject is deliberately *narrow* + # (include_convergent=False): the synonymous cousins are dropped, because VA-Spec carries no per-member + # provenance to mark them as unmeasured, and StudyResult.focusVariant already pins the concrete measured + # allele. + transit = build_categorical_variant(links, name=variant.urn or "", include_convergent=False) + if transit is not None and len(transit.categorical_variant.members) > 1: + subject_variant: MolecularVariation | CategoricalVariant = transit.categorical_variant + else: + subject_variant = vrs_object_from_mapped_variant(measured_allele.post_mapped) + + return VariantAnnotationContext( + variant=variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant, + as_of=as_of, + ) diff --git a/src/mavedb/lib/annotation/document.py b/src/mavedb/lib/annotation/document.py index 039142030..d9e58cfc2 100644 --- a/src/mavedb/lib/annotation/document.py +++ b/src/mavedb/lib/annotation/document.py @@ -7,8 +7,8 @@ from ga4gh.va_spec.base.core import Document from mavedb.constants import MAVEDB_FRONTEND_URL +from mavedb.models.allele import Allele from mavedb.models.experiment import Experiment -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -128,31 +128,34 @@ def score_calibration_as_document(score_calibration: ScoreCalibration) -> Docume ) -def mapped_variant_as_iri(mapped_variant: MappedVariant) -> Optional[IRI]: +def measured_allele_as_iri(allele: Allele) -> Optional[IRI]: """ - Create an IRI as described in for the provided MaveDB mapped variant. Within - the context of VA-Spec, these can be used interchangeably with an equivalent document object for brevity. + Create an IRI as described in for the + measured (authoritative) allele, keyed by its ClinGen allele id. Within the context of VA-Spec, these + can be used interchangeably with an equivalent document object for brevity. ``None`` when the allele + has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None - return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(mapped_variant.clingen_allele_id)}") + return IRI(f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}") -def mapped_variant_to_document(mapped_variant: MappedVariant) -> Optional[Document]: +def measured_allele_to_document(allele: Allele) -> Optional[Document]: """ Create a [VA Document](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/information-entities/document.html#document) - object from the provided MaveDB mapped variant. + object from the provided MaveDB measured (authoritative) allele, keyed by its ClinGen allele id. ``None`` + when the allele has no ClinGen allele id. """ - if not mapped_variant.clingen_allele_id: + if not allele.clingen_allele_id: return None return Document( - id=mapped_variant.variant.urn, - name="MaveDB Mapped Variant", - documentType="mapped genomic variant description", - # We only reach this point if a IRI is guaranteed to exist - urls=[mapped_variant_as_iri(mapped_variant).root], # type: ignore + id=allele.clingen_allele_id, + name="MaveDB Measured Allele", + documentType="measured allele", + # The measured allele document is keyed by its ClinGen allele id, so the IRI is guaranteed to exist. + urls=[measured_allele_as_iri(allele).root], # type: ignore ) diff --git a/src/mavedb/lib/annotation/eligibility.py b/src/mavedb/lib/annotation/eligibility.py new file mode 100644 index 000000000..8a6e05941 --- /dev/null +++ b/src/mavedb/lib/annotation/eligibility.py @@ -0,0 +1,128 @@ +"""Whether a variant is eligible for VA-Spec annotation. + +Gate predicates the annotation builders consult before constructing a statement: a variant needs a real +score (the base assumption) and its score set needs at least one calibration usable for the target +annotation type (:func:`calibration.score_calibration_may_be_used_for_annotation`). Split by annotation +type into ``can_annotate_variant_for_pathogenicity_evidence`` / ``can_annotate_variant_for_functional_statement``. +""" + +from typing import Literal + +from mavedb.lib.annotation.calibration import score_calibration_may_be_used_for_annotation +from mavedb.lib.variants import variant_score +from mavedb.models.variant import Variant + + +def _can_annotate_variant_base_assumptions(variant: Variant) -> bool: + """ + Check if a variant meets the basic requirements for annotation. + + This function validates that a variant has the necessary data to proceed with + annotation by checking for a valid score value. + + Args: + variant (Variant): The variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated (has a non-None numeric score), False otherwise. + """ + # A variant is annotatable only if it carries a real (non-null, numeric) score. + if variant_score(variant) is None: + return False + + return True + + +def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant: Variant, + annotation_type: Literal["pathogenicity", "functional"], + allow_research_use_only_calibrations: bool = False, +) -> bool: + """ + Check if a variant's score set contains any of the required calibrations for annotation. + + Args: + variant (Variant): The variant whose score set is checked. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. + Must be either "pathogenicity" or "functional". + allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as + research use only as valid for annotation. Defaults to False. + + Returns: + bool: True if the variant's score set contains at least one valid calibration with the required + classifications for the specified annotation type. False otherwise. + """ + if variant.score_set.score_calibrations is None: + return False + + return any( + score_calibration_may_be_used_for_annotation( + score_calibration, + annotation_type, + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + ) + for score_calibration in variant.score_set.score_calibrations + ) + + +def can_annotate_variant_for_pathogenicity_evidence( + variant: Variant, allow_research_use_only_calibrations=False +) -> bool: + """ + Determine if a variant can be annotated for pathogenicity evidence. + + This function checks if a variant meets all the necessary conditions to receive + pathogenicity evidence annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for pathogenicity evidence annotation. + + Args: + variant (Variant): The variant to evaluate for pathogenicity evidence annotation eligibility. + + Returns: + bool: True if the variant can be annotated for pathogenicity evidence, False otherwise. + + Notes: + The function performs two main validation checks: + 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. + + Both checks must pass for the variant to be considered eligible for + pathogenicity evidence annotation. + """ + if not _can_annotate_variant_base_assumptions(variant): + return False + if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations + ): + return False + + return True + + +def can_annotate_variant_for_functional_statement(variant: Variant, allow_research_use_only_calibrations=False) -> bool: + """ + Determine if a variant can be annotated for functional statements. + + This function checks if a variant meets all the necessary conditions to receive + functional annotations by validating base assumptions and ensuring the variant's + score calibrations contain the required kinds for functional annotation. + + Args: + variant (Variant): The variant to check for annotation eligibility. + + Returns: + bool: True if the variant can be annotated for functional statements, False otherwise. + + Notes: + The function performs two main checks: + 1. Validates base assumptions using _can_annotate_variant_base_assumptions + 2. Verifies score calibrations have an appropriate calibration for functional annotation. + """ + if not _can_annotate_variant_base_assumptions(variant): + return False + if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations + ): + return False + + return True diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..5ec2d5dcb 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -16,6 +16,7 @@ functional_classification_of_variant, pathogenicity_classification_of_variant, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -30,18 +31,17 @@ functional_score_calibration_as_method, pathogenicity_score_calibration_as_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration def acmg_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, proposition: VariantPathogenicityProposition, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> VariantPathogenicityEvidenceLine: containing_evidence_range, evidence_outcome, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, score_calibration + context.variant, score_calibration ) if not evidence_strength: @@ -63,7 +63,7 @@ def acmg_evidence_line( direction_of_evidence = direction_of_support_for_pathogenicity_classification(evidence_outcome) return VariantPathogenicityEvidenceLine( - description=f"Pathogenicity evidence line for {mapped_variant.variant.urn}.", + description=f"Pathogenicity evidence line for {context.variant.urn}.", hasEvidenceItems=list(evidence), specifiedBy=pathogenicity_score_calibration_as_method(score_calibration, evidence_outcome), evidenceOutcome={ @@ -77,7 +77,7 @@ def acmg_evidence_line( directionOfEvidenceProvided=direction_of_evidence, contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], targetProposition=proposition, @@ -93,14 +93,14 @@ def acmg_evidence_line( def functional_evidence_line( - mapped_variant: MappedVariant, + context: VariantAnnotationContext, score_calibration: ScoreCalibration, evidence: list[Union[StudyResult, EvidenceLineType, StatementType, iriReference]], ) -> EvidenceLine: - containing_evidence_range, classification = functional_classification_of_variant(mapped_variant, score_calibration) + containing_evidence_range, classification = functional_classification_of_variant(context.variant, score_calibration) return EvidenceLine( - description=f"Functional evidence line for {mapped_variant.variant.urn}", + description=f"Functional evidence line for {context.variant.urn}", hasEvidenceItems=[StudyResult(root=item) for item in evidence], directionOfEvidenceProvided=direction_of_support_for_functional_classification(classification), evidenceOutcome=MappableConcept( @@ -112,7 +112,7 @@ def functional_evidence_line( specifiedBy=functional_score_calibration_as_method(score_calibration), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], reportedIn=[score_calibration_as_document(score_calibration)], diff --git a/src/mavedb/lib/annotation/proposition.py b/src/mavedb/lib/annotation/proposition.py index 963327c73..143d5d608 100644 --- a/src/mavedb/lib/annotation/proposition.py +++ b/src/mavedb/lib/annotation/proposition.py @@ -2,22 +2,77 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition from mavedb.lib.annotation.condition import generic_disease_condition +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.document import experiment_to_document -from mavedb.lib.annotation.util import sequence_feature_for_mapped_variant, variation_from_mapped_variant -from mavedb.models.mapped_variant import MappedVariant +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.types.annotation import SequenceFeature +from mavedb.lib.variants import target_for_variant +from mavedb.models.variant import Variant -def mapped_variant_to_experimental_variant_clinical_impact_proposition( - mapped_variant: MappedVariant, +def sequence_feature_for_variant(variant: Variant) -> SequenceFeature: + """ + Extract the sequence feature (e.g., gene or transcript) associated with a variant. + + This function retrieves the sequence feature from the variant's target data, which is + necessary for generating annotations that reference specific genomic features. Co-located with the + propositions that consume it — its only caller. + + Args: + variant (Variant): The variant whose target gene supplies the sequence feature. + + Returns: + SequenceFeature: Named tuple with: + - `identifier`: sequence feature identifier (gene/transcript ID or name) + - `system`: source/system URL for the identifier + + """ + target = target_for_variant(variant) + if target is None: + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable target gene." + " Unable to extract sequence feature for annotation." + ) + + # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. + # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or + # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. + if target.mapped_hgnc_name: + return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") + + post_mapped_ids = extract_ids_from_post_mapped_metadata( + target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore + ) + if post_mapped_ids: + post_mapped_id = post_mapped_ids[0] + if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): + return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") + elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): + return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") + + return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") + + if target.name: + return SequenceFeature(target.name, "https://www.mavedb.org/") + + raise MappingDataDoesntExistException( + f"Variant {variant.urn} does not have an identifiable sequence feature in its target gene data." + " Unable to extract sequence feature for annotation." + ) + + +def variant_pathogenicity_proposition( + context: VariantAnnotationContext, ) -> VariantPathogenicityProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return VariantPathogenicityProposition( - description=f"Variant pathogenicity proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant pathogenicity proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="isCausalFor", objectCondition=generic_disease_condition(), geneContextQualifier=sequence_feature @@ -26,18 +81,18 @@ def mapped_variant_to_experimental_variant_clinical_impact_proposition( ) -def mapped_variant_to_experimental_variant_functional_impact_proposition( - mapped_variant: MappedVariant, +def variant_functional_impact_proposition( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactProposition: - coding, system = sequence_feature_for_mapped_variant(mapped_variant) + coding, system = sequence_feature_for_variant(context.variant) sequence_feature = MappableConcept( primaryCoding=Coding(code=coding, system=system), ) return ExperimentalVariantFunctionalImpactProposition( - description=f"Variant functional impact proposition for {mapped_variant.variant.urn}.", - subjectVariant=variation_from_mapped_variant(mapped_variant), + description=f"Variant functional impact proposition for {context.variant.urn}.", + subjectVariant=context.subject_variant, predicate="impactsFunctionOf", objectSequenceFeature=sequence_feature, - experimentalContextQualifier=experiment_to_document(mapped_variant.variant.score_set.experiment), + experimentalContextQualifier=experiment_to_document(context.variant.score_set.experiment), ) diff --git a/src/mavedb/lib/annotation/statement.py b/src/mavedb/lib/annotation/statement.py index c77d30f1a..7db3b05d8 100644 --- a/src/mavedb/lib/annotation/statement.py +++ b/src/mavedb/lib/annotation/statement.py @@ -10,6 +10,7 @@ ) from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_score_calibration_contribution, @@ -20,23 +21,22 @@ variant_interpretation_functional_guideline_method, variant_interpretation_pathogenicity_guideline_method, ) -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification -def mapped_variant_to_functional_statement( - mapped_variant: MappedVariant, +def functional_statement( + context: VariantAnnotationContext, proposition: ExperimentalVariantFunctionalImpactProposition, evidence: list[EvidenceLine], score_calibration: ScoreCalibration, functional_classification: ExperimentalVariantFunctionalImpactClassification, ) -> Statement: """ - Create a functional impact statement for a mapped variant. + Create a functional impact statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The functional impact proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -48,11 +48,11 @@ def mapped_variant_to_functional_statement( direction = aggregate_direction_of_evidence(evidence) return Statement( - description=f"Variant functional impact statement for {mapped_variant.variant.urn}.", + description=f"Variant functional impact statement for {context.variant.urn}.", specifiedBy=variant_interpretation_functional_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, @@ -67,18 +67,18 @@ def mapped_variant_to_functional_statement( ) -def mapped_variant_to_pathogenicity_statement( - mapped_variant: MappedVariant, +def pathogenicity_statement( + context: VariantAnnotationContext, proposition: VariantPathogenicityProposition, evidence: list[VariantPathogenicityEvidenceLine], score_calibration: ScoreCalibration, functional_range: Optional[ScoreCalibrationFunctionalClassification], ) -> VariantPathogenicityStatement: """ - Create a pathogenicity statement for a mapped variant. + Create a pathogenicity statement for a variant. Args: - mapped_variant: The variant being classified + context: The variant's annotation context (variant + mapping-record provenance) proposition: The pathogenicity proposition evidence: List of evidence lines supporting the statement score_calibration: The score calibration with the strongest evidence @@ -105,11 +105,11 @@ def mapped_variant_to_pathogenicity_statement( acmg_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE return VariantPathogenicityStatement( - description=f"Variant pathogenicity statement for {mapped_variant.variant.urn}.", + description=f"Variant pathogenicity statement for {context.variant.urn}.", specifiedBy=variant_interpretation_pathogenicity_guideline_method(), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), + mavedb_vrs_contribution(context), mavedb_score_calibration_contribution(score_calibration), ], proposition=proposition, diff --git a/src/mavedb/lib/annotation/study_result.py b/src/mavedb/lib/annotation/study_result.py index b93ea1fee..c45457468 100644 --- a/src/mavedb/lib/annotation/study_result.py +++ b/src/mavedb/lib/annotation/study_result.py @@ -1,5 +1,6 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.annotation.contribution import ( mavedb_api_contribution, mavedb_creator_contribution, @@ -7,31 +8,31 @@ mavedb_vrs_contribution, ) from mavedb.lib.annotation.dataset import score_set_to_data_set -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri from mavedb.lib.annotation.method import ( publication_identifiers_to_method, ) -from mavedb.lib.annotation.util import variation_from_mapped_variant +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.lib.variants import variant_score -from mavedb.models.mapped_variant import MappedVariant -def mapped_variant_to_experimental_variant_impact_study_result( - mapped_variant: MappedVariant, +def variant_impact_study_result( + context: VariantAnnotationContext, ) -> ExperimentalVariantFunctionalImpactStudyResult: + # The study result's focus is the concrete measured allele — never a CategoricalVariant (VA-Spec + # narrows ``focusVariant`` to MolecularVariation). The context guarantees a hydratable post_mapped. return ExperimentalVariantFunctionalImpactStudyResult( - description=f"Variant effect study result for {mapped_variant.variant.urn}.", - focusVariant=variation_from_mapped_variant(mapped_variant), - functionalImpactScore=variant_score(mapped_variant.variant), - specifiedBy=publication_identifiers_to_method( - mapped_variant.variant.score_set.publication_identifier_associations - ), - sourceDataSet=score_set_to_data_set(mapped_variant.variant.score_set), + description=f"Variant effect study result for {context.variant.urn}.", + # post_mapped is guaranteed non-null by variant_annotation_context (it returns None otherwise). + focusVariant=vrs_object_from_mapped_variant(context.measured_allele.post_mapped), # type: ignore[arg-type] + functionalImpactScore=variant_score(context.variant), + specifiedBy=publication_identifiers_to_method(context.variant.score_set.publication_identifier_associations), + sourceDataSet=score_set_to_data_set(context.variant.score_set), contributions=[ mavedb_api_contribution(), - mavedb_vrs_contribution(mapped_variant), - mavedb_creator_contribution(mapped_variant.variant, mapped_variant.variant.score_set.created_by), - mavedb_modifier_contribution(mapped_variant.variant, mapped_variant.variant.score_set.modified_by), + mavedb_vrs_contribution(context), + mavedb_creator_contribution(context.variant, context.variant.score_set.created_by), + mavedb_modifier_contribution(context.variant, context.variant.score_set.modified_by), ], - reportedIn=filter(None, [variant_as_iri(mapped_variant.variant), mapped_variant_as_iri(mapped_variant)]), + reportedIn=filter(None, [variant_as_iri(context.variant), measured_allele_as_iri(context.measured_allele)]), ) diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py deleted file mode 100644 index ed90c5d03..000000000 --- a/src/mavedb/lib/annotation/util.py +++ /dev/null @@ -1,504 +0,0 @@ -from typing import Literal, Optional - -from ga4gh.core.models import Extension -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided -from ga4gh.vrs.models import ( - Allele, - CisPhasedBlock, - Expression, - LengthExpression, - LiteralSequenceExpression, - MolecularVariation, - ReferenceLengthExpression, - SequenceLocation, - SequenceReference, -) - -from mavedb.lib.annotation.classification import ( - ExperimentalVariantFunctionalImpactClassification, - functional_classification_of_variant, - pathogenicity_classification_of_variant, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata -from mavedb.lib.types.annotation import SequenceFeature -from mavedb.lib.variants import target_for_variant, variant_score -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_calibration import ScoreCalibration -from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification - - -def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: - """ - Converts a dictionary containing allelic mapping results into an Allele object. - - This function handles the possibility of an extra nesting level in early VRS 1.3 objects, - where Allele objects are contained within a `variation` property. If the `variation` key - is not present, the function assumes the dictionary itself represents the variation. - - Args: - allelic_mapping_results (dict): A dictionary containing allelic mapping results. - It may include a `variation` key or directly represent the variation. - - Returns: - Allele: An Allele object constructed from the provided mapping results. - - Raises: - KeyError: If required keys are missing from the input dictionary. - """ - - # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects - # are contained in a `variation` property. Although it's unlikely variants of this form - # will ever be exported in this format, we handle the possibility. - try: - variation = allelic_mapping_results["variation"] - except KeyError: - variation = allelic_mapping_results - - state_dict = variation["state"] - state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression - if state_dict.get("type") == "ReferenceLengthExpression": - state = ReferenceLengthExpression(**state_dict) - elif state_dict.get("type") == "LengthExpression": - state = LengthExpression(**state_dict) - elif state_dict.get("type") == "LiteralSequenceExpression": - state = LiteralSequenceExpression(**state_dict) - else: - raise ValueError( - f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " - "Update allele_from_mapped_variant_dictionary_result to handle this type." - ) - - # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on - # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid, so using model_validate() - # directly is not possible against stored mapping results. - return Allele( - id=variation.get("id"), - state=state, - digest=variation.get("digest"), - location=SequenceLocation( - start=variation.get("location", {}).get("start"), - end=variation.get("location", {}).get("end"), - digest=variation.get("location", {}).get("digest"), - id=variation.get("location", {}).get("id"), - sequenceReference=SequenceReference( - name=variation.get("location", {}).get("sequenceReference", {}).get("name"), - refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), - ), - ), - extensions=[ - Extension( - id=extension.get("id"), - name=extension["name"], - description=extension.get("description"), - value=extension.get("value"), - ) - for extension in variation.get("extensions", []) - ], - expressions=[ - Expression( - id=expression.get("id"), - syntax=expression["syntax"], - syntax_version=expression.get("syntax_version"), - value=expression["value"], - ) - for expression in variation.get("expressions", []) - ], - ) - - -def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: - """ - Extracts a VRS (Variation Representation Specification) object from a mapped variant. - - This function processes a dictionary of mapping results and returns a VRS object, - which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object - returned depends on the "type" field in the input dictionary. - - Args: - mapping_results (dict): A dictionary containing the mapping results of a variant. - It must include a "type" key indicating the type of VRS object - ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). - If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also - include a "members" key containing a list of member alleles. - - Returns: - MolecularVariation: A VRS object representing the mapped variant. This will be - either a `CisPhasedBlock` containing its member variants or an `Allele` - derived from the mapping results. - - Raises: - KeyError: If required keys are missing from the `mapping_results` dictionary. - """ - if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": - return MolecularVariation( - # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) - CisPhasedBlock( - id=None, - members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], - ) - ) - - return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) - - -def variation_from_mapped_variant(mapped_variant: MappedVariant) -> MolecularVariation: - """ - Converts mapping results from a mapped variant into a MolecularVariation object. - - This function takes a `MappedVariant` object and extracts its post-mapped - variant to generate a corresponding `MolecularVariation` object. If the - `post_mapped` attribute of the `MappedVariant` is `None`, an exception is raised. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing - the variant data. - - Returns: - MolecularVariation: The molecular variation derived from the post-mapped - variant. - - Raises: - MappingDataDoesntExistException: If the `post_mapped` attribute of the - `mapped_variant` is `None`, indicating that the post-mapped variant - data is unavailable. - """ - if mapped_variant.post_mapped is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have a post mapped variant." - " Unable to extract variation data." - ) - - return vrs_object_from_mapped_variant(mapped_variant.post_mapped) - - -def _can_annotate_variant_base_assumptions(mapped_variant: MappedVariant) -> bool: - """ - Check if a mapped variant meets the basic requirements for annotation. - - This function validates that a mapped variant has the necessary data - to proceed with annotation by checking for a valid score value. - - Args: - mapped_variant (MappedVariant): The mapped variant to check for - annotation eligibility. - - Returns: - bool: True if the variant can be annotated (has score ranges and - a non-None score), False otherwise. - """ - # A variant is annotatable only if it carries a real (non-null, numeric) score. - if variant_score(mapped_variant.variant) is None: - return False - - return True - - -def score_calibration_may_be_used_for_annotation( - score_calibration: ScoreCalibration, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a score calibration may be used for annotation based on its properties. - - This function evaluates whether a given score calibration is suitable for use in - annotation by checking its research use only status and the presence of required - classifications based on the annotation type. - - Args: - score_calibration (ScoreCalibration): The score calibration to evaluate. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation - being considered, which determines the required classifications for validity. - allow_research_use_only_calibrations (bool, optional): Whether to allow calibrations - marked as research use only for annotation. Defaults to False. - - Returns: - bool: True if the score calibration can be used for annotation, False otherwise. - """ - if score_calibration.research_use_only and not allow_research_use_only_calibrations: - return False - - if score_calibration.functional_classifications is None or len(score_calibration.functional_classifications) == 0: - return False - - if annotation_type == "pathogenicity" and all( - fr.acmg_classification is None for fr in score_calibration.functional_classifications - ): - return False - - return True - - -def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant: MappedVariant, - annotation_type: Literal["pathogenicity", "functional"], - allow_research_use_only_calibrations: bool = False, -) -> bool: - """ - Check if a mapped variant's score set contains any of the required calibrations for annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant with score set data. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. - Must be either "pathogenicity" or "functional". - allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as - research use only as valid for annotation. Defaults to False. - - Returns: - bool: True if the variant's score set contains at least one valid calibration with the required - classifications for the specified annotation type. False otherwise. - """ - if mapped_variant.variant.score_set.score_calibrations is None: - return False - - return any( - score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type, - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ) - for score_calibration in mapped_variant.variant.score_set.score_calibrations - ) - - -def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False -) -> bool: - """ - Determine if a mapped variant can be annotated for pathogenicity evidence. - - This function checks if a variant meets all the necessary conditions to receive - pathogenicity evidence annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for pathogenicity evidence annotation. - - Args: - mapped_variant (MappedVariant): The mapped variant object to evaluate - for pathogenicity evidence annotation eligibility. - - Returns: - bool: True if the variant can be annotated for pathogenicity evidence, - False otherwise. - - Notes: - The function performs two main validation checks: - 1. Basic annotation assumptions via _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for pathogenicity evidence annotation. - - Both checks must pass for the variant to be considered eligible for - pathogenicity evidence annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - - return True - - -def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False -) -> bool: - """ - Determine if a mapped variant can be annotated for functional statements. - - This function checks if a variant meets all the necessary conditions to receive - functional annotations by validating base assumptions and ensuring the variant's - score calibrations contain the required kinds for functional annotation. - - Args: - mapped_variant (MappedVariant): The variant object to check for annotation - eligibility, containing mapping information and score data. - - Returns: - bool: True if the variant can be annotated for functional statements, - False otherwise. - - Notes: - The function performs two main checks: - 1. Validates base assumptions using _can_annotate_variant_base_assumptions - 2. Verifies score calibrations have an appropriate calibration for functional annotation. - """ - if not _can_annotate_variant_base_assumptions(mapped_variant): - return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - - return True - - -def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: - """ - Extract the sequence feature (e.g., gene or transcript) associated with a mapped variant. - - This function retrieves the sequence feature from the mapped variant's data, which is - necessary for generating annotations that reference specific genomic features. - - Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant data. - - Returns: - SequenceFeature: Named tuple with: - - `identifier`: sequence feature identifier (gene/transcript ID or name) - - `system`: source/system URL for the identifier - - """ - target = target_for_variant(mapped_variant.variant) - if target is None: - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable target gene." - " Unable to extract sequence feature for annotation." - ) - - # Prefer the mapped HGNC name if it's available, as this is more likely to be stable and recognizable than accessions or other identifiers. - # If the mapped HGNC name is not available, fall back to extracting an identifier from the post-mapped metadata, which may be a gene or - # transcript identifier of varying formats. If neither of those options are available, fall back to the target gene's name as listed in MaveDB. - if target.mapped_hgnc_name: - return SequenceFeature(target.mapped_hgnc_name, "https://www.genenames.org/") - - post_mapped_ids = extract_ids_from_post_mapped_metadata( - target.post_mapped_metadata if target.post_mapped_metadata else {} # type: ignore - ) - if post_mapped_ids: - post_mapped_id = post_mapped_ids[0] - if post_mapped_id.startswith("ENSG") or post_mapped_id.startswith("ENST") or post_mapped_id.startswith("ENSP"): - return SequenceFeature(post_mapped_id, "https://www.ensembl.org/index.html") - elif post_mapped_id.startswith("NM_") or post_mapped_id.startswith("NR_") or post_mapped_id.startswith("NP_"): - return SequenceFeature(post_mapped_id, "https://www.ncbi.nlm.nih.gov/refseq/") - - return SequenceFeature(post_mapped_id, "transcript or gene identifier of unknown source") - - if target.name: - return SequenceFeature(target.name, "https://www.mavedb.org/") - - raise MappingDataDoesntExistException( - f"Variant {mapped_variant.variant.urn} does not have an identifiable sequence feature in its target gene data." - " Unable to extract sequence feature for annotation." - ) - - -def select_strongest_functional_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for functional classification. - - In case of ties or conflicting classifications, defaults to normal classification. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should be classified as INDETERMINATE but still receive annotations. - """ - if not calibrations: - return None, None - - # Collect all calibrations and their classifications - candidates: list[ - tuple[ - ScoreCalibration, - ScoreCalibrationFunctionalClassification, - ExperimentalVariantFunctionalImpactClassification, - ] - ] = [] - - for calibration in calibrations: - functional_range, classification = functional_classification_of_variant(mapped_variant, calibration) - if functional_range is not None: - candidates.append((calibration, functional_range, classification)) - - # If variant is not in any range, return first calibration with None to indicate INDETERMINATE - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Check if all classifications agree - classifications = [c[2] for c in candidates] - if all(cls == classifications[0] for cls in classifications): - # All agree, return the first one - return candidates[0][0], candidates[0][1] - - # Conflict exists: default to normal classification - normal_candidates = [c for c in candidates if c[2] == ExperimentalVariantFunctionalImpactClassification.NORMAL] - if normal_candidates: - return normal_candidates[0][0], normal_candidates[0][1] - - # If no normal classification, return the first candidate - return candidates[0][0], candidates[0][1] - - -def select_strongest_pathogenicity_calibration( - mapped_variant: MappedVariant, - calibrations: list[ScoreCalibration], -) -> tuple[Optional[ScoreCalibration], Optional[ScoreCalibrationFunctionalClassification]]: - """ - Select the calibration with the strongest evidence for pathogenicity classification. - - Uses ACMG evidence strength to determine the strongest evidence. - In case of ties with conflicting evidence (both benign and pathogenic), defaults to uncertain - significance by returning None for the functional range. - Returns the calibration and its functional classification range that contains the variant. - - If the variant is not contained in any range, returns (first_calibration, None) to indicate - the variant should receive annotations even though it's not classified in any range. - """ - if not calibrations: - return None, None - - # Define evidence strength ordering (higher index = stronger evidence) - # Note: VA-Spec StrengthOfEvidenceProvided doesn't have MODERATE_PLUS, only our MaveDB enum does. - # The classification.py module maps MODERATE_PLUS to MODERATE when returning VA-Spec enum values. - strength_order = { - None: 0, - VaSpecStrengthOfEvidenceProvided.SUPPORTING: 1, - VaSpecStrengthOfEvidenceProvided.MODERATE: 2, - VaSpecStrengthOfEvidenceProvided.STRONG: 3, - VaSpecStrengthOfEvidenceProvided.VERY_STRONG: 4, - } - - # Collect all calibrations with their evidence strength and criterion - candidates: list[tuple[ScoreCalibration, ScoreCalibrationFunctionalClassification, int, bool]] = [] - - for calibration in calibrations: - functional_range, criterion, evidence_strength = pathogenicity_classification_of_variant( - mapped_variant, calibration - ) - if functional_range is not None: - strength_value = strength_order.get(evidence_strength, 0) - is_benign = criterion.name.startswith("B") if criterion else False - candidates.append((calibration, functional_range, strength_value, is_benign)) - - # If variant is not in any range, return first calibration with None - if not candidates: - return calibrations[0], None - - # If only one candidate, return it - if len(candidates) == 1: - return candidates[0][0], candidates[0][1] - - # Find the maximum strength - max_strength = max(c[2] for c in candidates) - strongest_candidates = [c for c in candidates if c[2] == max_strength] - - # If only one with max strength, return it - if len(strongest_candidates) == 1: - return strongest_candidates[0][0], strongest_candidates[0][1] - - # Tie: check for conflicting evidence (both benign and pathogenic) - has_benign = any(c[3] for c in strongest_candidates) - has_pathogenic = any(not c[3] for c in strongest_candidates) - - # If there's a conflict between benign and pathogenic evidence of equal strength, - # return None for the functional range to indicate uncertain significance - if has_benign and has_pathogenic: - return strongest_candidates[0][0], None - - # Otherwise return the first of the strongest - return strongest_candidates[0][0], strongest_candidates[0][1] diff --git a/src/mavedb/lib/vrs.py b/src/mavedb/lib/vrs.py new file mode 100644 index 000000000..c5ee00ee1 --- /dev/null +++ b/src/mavedb/lib/vrs.py @@ -0,0 +1,134 @@ +"""Deserialization of stored ``post_mapped`` JSONB into GA4GH VRS objects. + +The mapping pipeline writes each allele's mapped representation as a VRS-shaped dict in +``Allele.post_mapped``. These helpers rehydrate that dict into strict ``ga4gh.vrs`` Pydantic models for the +Cat-VRS transit, the variant-detail envelope, and the VA-Spec annotation subjects. +""" + +from ga4gh.core.models import Extension +from ga4gh.vrs.models import ( + Allele, + CisPhasedBlock, + Expression, + LengthExpression, + LiteralSequenceExpression, + MolecularVariation, + ReferenceLengthExpression, + SequenceLocation, + SequenceReference, +) + + +def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: + """ + Converts a dictionary containing allelic mapping results into an Allele object. + + This function handles the possibility of an extra nesting level in early VRS 1.3 objects, + where Allele objects are contained within a `variation` property. If the `variation` key + is not present, the function assumes the dictionary itself represents the variation. + + Args: + allelic_mapping_results (dict): A dictionary containing allelic mapping results. + It may include a `variation` key or directly represent the variation. + + Returns: + Allele: An Allele object constructed from the provided mapping results. + + Raises: + KeyError: If required keys are missing from the input dictionary. + """ + + # NOTE: Early VRS 1.3 objects may contain an extra nesting level, where Allele objects + # are contained in a `variation` property. Although it's unlikely variants of this form + # will ever be exported in this format, we handle the possibility. + try: + variation = allelic_mapping_results["variation"] + except KeyError: + variation = allelic_mapping_results + + state_dict = variation["state"] + state: ReferenceLengthExpression | LengthExpression | LiteralSequenceExpression + if state_dict.get("type") == "ReferenceLengthExpression": + state = ReferenceLengthExpression(**state_dict) + elif state_dict.get("type") == "LengthExpression": + state = LengthExpression(**state_dict) + elif state_dict.get("type") == "LiteralSequenceExpression": + state = LiteralSequenceExpression(**state_dict) + else: + raise ValueError( + f"Unsupported VRS Allele state type {state_dict.get('type')!r}. " + "Update allele_from_mapped_variant_dictionary_result to handle this type." + ) + + # Mapping results were not guaranteed to be generated on this version of VRS. + # Explicit field extraction for alleles is intentional: stored dicts may contain extra fields (e.g. "type" on + # Extension, "label" on SequenceReference) that the strict VRS Pydantic models forbid. In such cases, + # using model_validate() directly is not possible. + return Allele( + id=variation.get("id"), + state=state, + digest=variation.get("digest"), + location=SequenceLocation( + start=variation.get("location", {}).get("start"), + end=variation.get("location", {}).get("end"), + digest=variation.get("location", {}).get("digest"), + id=variation.get("location", {}).get("id"), + sequenceReference=SequenceReference( + name=variation.get("location", {}).get("sequenceReference", {}).get("name"), + refgetAccession=variation.get("location", {}).get("sequenceReference", {}).get("refgetAccession"), + ), + ), + extensions=[ + Extension( + id=extension.get("id"), + name=extension["name"], + description=extension.get("description"), + value=extension.get("value"), + ) + for extension in variation.get("extensions", []) + ], + expressions=[ + Expression( + id=expression.get("id"), + syntax=expression["syntax"], + syntax_version=expression.get("syntax_version"), + value=expression["value"], + ) + for expression in variation.get("expressions", []) + ], + ) + + +def vrs_object_from_mapped_variant(mapping_results: dict) -> MolecularVariation: + """ + Extracts a VRS (Variation Representation Specification) object from a stored ``post_mapped`` dict. + + This function processes a dictionary of mapping results and returns a VRS object, + which can either be an `Allele` or a `CisPhasedBlock`. The type of VRS object + returned depends on the "type" field in the input dictionary. + + Args: + mapping_results (dict): A dictionary containing the mapping results of a variant. + It must include a "type" key indicating the type of VRS object + ("CisPhasedBlock" or "Haplotype" for a CisPhasedBlock, or "Allele"). + If the type is "CisPhasedBlock" or "Haplotype", the dictionary must also + include a "members" key containing a list of member alleles. + + Returns: + MolecularVariation: A VRS object representing the mapped variant. This will be + either a `CisPhasedBlock` containing its member variants or an `Allele` + derived from the mapping results. + + Raises: + KeyError: If required keys are missing from the `mapping_results` dictionary. + """ + if mapping_results.get("type") == "CisPhasedBlock" or mapping_results.get("type") == "Haplotype": + return MolecularVariation( + # It's unclear why MyPy complains about the missing id field, so just add it as None (it is None by default anyway) + CisPhasedBlock( + id=None, + members=[allele_from_mapped_variant_dictionary_result(member) for member in mapping_results["members"]], + ) + ) + + return MolecularVariation(allele_from_mapped_variant_dictionary_result(mapping_results)) diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 851f6fcf0..f9a9480fa 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,8 +5,12 @@ including mock objects with proper calibrations and configurations. """ +from types import SimpleNamespace + import pytest +from mavedb.lib.annotation.context import VariantAnnotationContext +from mavedb.lib.vrs import vrs_object_from_mapped_variant from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_mapped_variant_with_functional_calibration_score_set, @@ -14,6 +18,32 @@ ) +def annotation_context_for(mapped_variant, subject_variant=None) -> VariantAnnotationContext: + """Build a DB-free ``VariantAnnotationContext`` around a mock mapped variant. + + The annotation builders read a context's ``variant`` (scores/calibrations), ``record`` (VRS-mapper + provenance), ``measured_allele`` (study-result focus + ClinGen IRI), and ``subject_variant`` (the VA + proposition subject). We synthesize the record/allele from the mock's mapping fields; the subject + defaults to the concrete measured variation, matching a non-projected variant. Mutating + ``context.variant`` mutates the underlying mock variant, since it is the same object. + """ + record = SimpleNamespace( + mapping_api_version=mapped_variant.mapping_api_version, + mapped_date=mapped_variant.mapped_date, + ) + measured_allele = SimpleNamespace( + post_mapped=mapped_variant.post_mapped, + clingen_allele_id=mapped_variant.clingen_allele_id, + ) + return VariantAnnotationContext( + variant=mapped_variant.variant, + record=record, + measured_allele=measured_allele, + subject_variant=subject_variant or vrs_object_from_mapped_variant(mapped_variant.post_mapped), + as_of=None, + ) + + @pytest.fixture def mock_mapped_variant(): """Override main fixture with properly configured mock for annotation tests.""" @@ -30,3 +60,25 @@ def mock_mapped_variant_with_functional_calibration_score_set(): def mock_mapped_variant_with_pathogenicity_calibration_score_set(): """Fixture for mock mapped variant with pathogenicity calibration score set.""" return create_mock_mapped_variant_with_pathogenicity_calibration_score_set(clingen_allele_id="CA123456") + + +@pytest.fixture +def mock_annotation_context(mock_mapped_variant) -> VariantAnnotationContext: + """A DB-free annotation context for an uncalibrated variant.""" + return annotation_context_for(mock_mapped_variant) + + +@pytest.fixture +def mock_annotation_context_with_functional_calibration_score_set( + mock_mapped_variant_with_functional_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a functional calibration.""" + return annotation_context_for(mock_mapped_variant_with_functional_calibration_score_set) + + +@pytest.fixture +def mock_annotation_context_with_pathogenicity_calibration_score_set( + mock_mapped_variant_with_pathogenicity_calibration_score_set, +) -> VariantAnnotationContext: + """A DB-free annotation context whose variant carries a pathogenicity calibration.""" + return annotation_context_for(mock_mapped_variant_with_pathogenicity_calibration_score_set) diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index b05c2c18b..eca223df3 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -2,7 +2,9 @@ Tests for mavedb.lib.annotation.annotate module. This module tests the main annotation functions that create statements and study results -for variants, focusing on object structure and validation. +for variants, focusing on object structure and validation. The builders take a DB-free +``VariantAnnotationContext`` (assembled by the ``mock_annotation_context*`` fixtures); the real +substrate-backed context build is covered by the integration tests in ``test_util.py``. """ # ruff: noqa: E402 @@ -25,9 +27,9 @@ class TestVariantStudyResult: """Unit tests for variant study result creation.""" - def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): + def test_variant_study_result_creates_valid_result(self, mock_annotation_context): """Test that variant study result creates a valid result object.""" - result = variant_study_result(mock_mapped_variant) + result = variant_study_result(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" @@ -37,35 +39,35 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): class TestVariantFunctionalImpactStatement: """Unit tests for variant functional impact statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_functional_impact_statement(mock_mapped_variant) + result = variant_functional_impact_statement(mock_annotation_context) assert result is None def test_only_research_use_only_calibrations_returns_none( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_functional_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_functional_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_functional_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + context = mock_annotation_context_with_functional_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_functional_impact_statement(context) assert result is None - def test_valid_statement_creation(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_valid_statement_creation(self, mock_annotation_context_with_functional_calibration_score_set): """Test creating valid functional impact statement with proper structure.""" - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" @@ -77,38 +79,37 @@ def test_valid_statement_creation(self, mock_mapped_variant_with_functional_cali ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that research-use-only calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_indeterminate( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that variant not in any functional range gets INDETERMINATE classification.""" from unittest.mock import patch from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set + context = mock_annotation_context_with_functional_calibration_score_set # Mock functional_classification_of_variant to return None range (variant not in any range) with patch( "mavedb.lib.annotation.annotate.functional_classification_of_variant", return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ): - result = variant_functional_impact_statement(mapped_variant) + result = variant_functional_impact_statement(context) assert result is not None assert result.type == "Statement" @@ -120,49 +121,52 @@ def test_variant_not_in_any_range_returns_indeterminate( class TestVariantPathogenicityStatement: """Unit tests for variant pathogenicity statement creation.""" - def test_no_calibrations_returns_none(self, mock_mapped_variant): + def test_no_calibrations_returns_none(self, mock_annotation_context): """Test that statement returns None when no calibrations exist.""" - result = variant_pathogenicity_statement(mock_mapped_variant) + result = variant_pathogenicity_statement(mock_annotation_context) assert result is None - def test_no_score_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_score_returns_none(self, mock_annotation_context_with_pathogenicity_calibration_score_set): """Test that statement returns None when variant has no score.""" - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.data = {"score_data": {"score": None}} - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + context = mock_annotation_context_with_pathogenicity_calibration_score_set + context.variant.data = {"score_data": {"score": None}} + result = variant_pathogenicity_statement(context) assert result is None def test_only_research_use_only_calibration_returns_none( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that statement returns None when only research use only primary calibrations exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Set all calibrations to research use only - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: calibration.research_use_only = True - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_no_acmg_classifications_returns_none(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_no_acmg_classifications_returns_none( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test that statement returns None when no ACMG classifications exist.""" + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Remove ACMG classifications from all calibrations - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + for calibration in context.variant.score_set.score_calibrations: acmg_removed = [deepcopy(r) for r in calibration.functional_classifications] for functional_classification in acmg_removed: functional_classification["acmgClassification"] = None calibration.functional_classifications = acmg_removed - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is None - def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + def test_valid_pathogenicity_statement_creation( + self, mock_annotation_context_with_pathogenicity_calibration_score_set + ): """Test creating valid pathogenicity statement with proper structure.""" - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.proposition.type == "VariantPathogenicityProposition" @@ -186,27 +190,27 @@ def test_valid_pathogenicity_statement_creation(self, mock_mapped_variant_with_p ) def test_skips_research_use_only_calibrations_when_mixed( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that research-use-only pathogenicity calibrations are skipped when mixed with regular calibrations.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] mixed_calibrations[0].research_use_only = True mixed_calibrations[1].research_use_only = False - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_functional_annotation( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test that functional annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_functional_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid: no functional classifications @@ -214,20 +218,19 @@ def test_skips_invalid_calibrations_when_functional_annotation( # Valid: retain default functional classifications mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_functional_impact_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_skips_invalid_calibrations_when_pathogenicity_annotation( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that pathogenicity annotation skips calibrations invalid under score_calibration_may_be_used_for_annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + context = mock_annotation_context_with_pathogenicity_calibration_score_set + calibrations = context.variant.score_set.score_calibrations mixed_calibrations = [deepcopy(calibrations[0]), deepcopy(calibrations[0])] # Invalid calibration: no functional classifications @@ -236,17 +239,15 @@ def test_skips_invalid_calibrations_when_pathogenicity_annotation( # Valid calibration retained mixed_calibrations[1].functional_classifications = deepcopy(calibrations[0].functional_classifications) - mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations = ( - mixed_calibrations - ) + context.variant.score_set.score_calibrations = mixed_calibrations - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(context) assert result is not None assert len(result.hasEvidenceLines) == 1 def test_variant_not_in_any_range_returns_uncertain_significance( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Test that variant not in any range gets UNCERTAIN_SIGNIFICANCE classification.""" from unittest.mock import patch @@ -255,7 +256,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set + context = mock_annotation_context_with_pathogenicity_calibration_score_set # Mock both classification functions to return None range (variant not in any range) with ( @@ -264,11 +265,11 @@ def test_variant_not_in_any_range_returns_uncertain_significance( return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), ), patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), ), ): - result = variant_pathogenicity_statement(mapped_variant) + result = variant_pathogenicity_statement(context) assert result is not None assert result.type == "Statement" @@ -276,7 +277,7 @@ def test_variant_not_in_any_range_returns_uncertain_significance( assert result.classification.primaryCoding.code.root == "uncertain significance" def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): """Regression test: hasEvidenceItems on VariantPathogenicityEvidenceLine must be model instances. @@ -284,7 +285,7 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( VariantPathogenicityEvidenceLine validation to fail when reconstructing nested VRS objects (e.g. Allele with production genomic coordinates). Model instances must be stored directly. """ - result = variant_pathogenicity_statement(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_pathogenicity_statement(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None for evidence_line in result.hasEvidenceLines: @@ -301,32 +302,28 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( class TestVariantHighestLevelAnnotation: """Unit tests for the highest-materialized-layer resolver used by the public data dump.""" - def test_study_result_when_uncalibrated(self, mock_mapped_variant): - result = variant_highest_level_annotation(mock_mapped_variant) + def test_study_result_when_uncalibrated(self, mock_annotation_context): + result = variant_highest_level_annotation(mock_annotation_context) assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" - def test_functional_statement_when_functional_only(self, mock_mapped_variant_with_functional_calibration_score_set): + def test_functional_statement_when_functional_only( + self, mock_annotation_context_with_functional_calibration_score_set + ): # The functional calibration fixture has no ACMG classifications, so the variant qualifies for the # functional layer but not pathogenicity. - result = variant_highest_level_annotation(mock_mapped_variant_with_functional_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_functional_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "ExperimentalVariantFunctionalImpactProposition" def test_pathogenicity_statement_when_calibrated( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set + self, mock_annotation_context_with_pathogenicity_calibration_score_set ): - result = variant_highest_level_annotation(mock_mapped_variant_with_pathogenicity_calibration_score_set) + result = variant_highest_level_annotation(mock_annotation_context_with_pathogenicity_calibration_score_set) assert result is not None assert result.type == "Statement" assert result.proposition.type == "VariantPathogenicityProposition" - - def test_none_when_unmapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - result = variant_highest_level_annotation(mock_mapped_variant) - assert result is None diff --git a/tests/lib/annotation/test_calibration.py b/tests/lib/annotation/test_calibration.py new file mode 100644 index 000000000..1614ce819 --- /dev/null +++ b/tests/lib/annotation/test_calibration.py @@ -0,0 +1,344 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.calibration — calibration eligibility + strongest-evidence selection.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.calibration import ( + score_calibration_may_be_used_for_annotation, + select_strongest_functional_calibration, + select_strongest_pathogenicity_calibration, +) + + +@pytest.mark.unit +class TestScoreCalibrationMayBeUsedForAnnotation: + def test_returns_false_for_research_use_only_when_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=False, + ) + is False + ) + + def test_returns_true_for_research_use_only_when_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.research_use_only = True + + assert ( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type="functional", + allow_research_use_only_calibrations=True, + ) + is True + ) + + def test_returns_false_when_functional_classifications_missing( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] + calibration.functional_classifications = [] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False + + def test_returns_false_for_pathogenicity_without_acmg_classifications( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for functional_classification in acmg_removed: + functional_classification["acmgClassification"] = None + calibration.functional_classifications = acmg_removed + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False + + def test_returns_true_for_pathogenicity_with_any_acmg_classification( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ + 0 + ] + + assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True + + +@pytest.mark.unit +class TestSelectStrongestFunctionalCalibrationUnit: + """Unit tests for select_strongest_functional_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant.variant, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that single calibration is returned.""" + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that first calibration is returned when all have same classification.""" + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + # Create multiple calibrations with same classification + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] # Should return the first one + + def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): + """Test that normal classification is preferred when there are conflicts.""" + from unittest.mock import MagicMock, patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different classifications + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + side_effect=[ + (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), + (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), + ], + ): + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + # Should return the normal classification (second one) + assert calibration == calibration2 + assert functional_range.label == "Normal" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification + + variant = mock_mapped_variant_with_functional_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + # Mock to return None range but INDETERMINATE classification (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.functional_classification_of_variant", + return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), + ): + calibration, functional_range = select_strongest_functional_calibration(variant, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None + + +@pytest.mark.unit +class TestSelectStrongestPathogenicityCalibrationUnit: + """Unit tests for select_strongest_pathogenicity_calibration function.""" + + def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): + """Test that empty calibration list returns None.""" + calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant.variant, []) + assert calibration is None + assert functional_range is None + + def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that single calibration is returned.""" + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + assert calibration is not None + assert calibration == calibrations[0] + assert functional_range is not None + + def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that calibration with strongest evidence is selected.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return different evidence strengths + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ( + MagicMock(label="Strong"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.VERY_STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return the one with VERY_STRONG evidence + assert calibration == calibration2 + assert functional_range.label == "Strong" + + def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength but different criteria (pathogenic vs benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Pathogenic"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration but None for range to indicate uncertain significance + assert calibration == calibration1 + assert functional_range is None + + def test_returns_classification_when_all_tied_are_same_type( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test that classification is returned normally when all tied candidates are the same type.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock to return same evidence strength and same type of criteria (both benign) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + ( + MagicMock(label="Benign1"), + VariantPathogenicityEvidenceLine.Criterion.BS3, + StrengthOfEvidenceProvided.STRONG, + ), + ( + MagicMock(label="Benign2"), + VariantPathogenicityEvidenceLine.Criterion.BP1, + StrengthOfEvidenceProvided.STRONG, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration with its range (no conflict, so normal classification) + assert calibration == calibration1 + assert functional_range.label == "Benign1" + + def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): + """Test that None evidence strength is handled correctly.""" + from unittest.mock import MagicMock, patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibration1 = variant.score_set.score_calibrations[0] + calibration2 = deepcopy(calibration1) + calibration2.id = 999 + calibrations = [calibration1, calibration2] + + # Mock with None and actual strength + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + side_effect=[ + (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ( + MagicMock(label="Moderate"), + VariantPathogenicityEvidenceLine.Criterion.PS3, + StrengthOfEvidenceProvided.MODERATE, + ), + ], + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return the one with actual strength (second) + assert calibration == calibration2 + assert functional_range.label == "Moderate" + + def test_returns_first_calibration_when_no_variants_in_ranges( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test that first calibration with None range is returned when variant is not in any functional range.""" + from unittest.mock import patch + + from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine + + variant = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + calibrations = variant.score_set.score_calibrations + + # Mock to return None range for all calibrations (variant not in any range) + with patch( + "mavedb.lib.annotation.calibration.pathogenicity_classification_of_variant", + return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), + ): + calibration, functional_range = select_strongest_pathogenicity_calibration(variant, calibrations) + + # Should return first calibration with None range (indicating variant not in any range) + assert calibration == calibrations[0] + assert functional_range is None diff --git a/tests/lib/annotation/test_classification.py b/tests/lib/annotation/test_classification.py index 82cd90684..fdbaf23cb 100644 --- a/tests/lib/annotation/test_classification.py +++ b/tests/lib/annotation/test_classification.py @@ -63,7 +63,7 @@ def test_normal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.normal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -76,7 +76,7 @@ def test_abnormal_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.ABNORMAL @@ -89,7 +89,7 @@ def test_indeterminate_classification(self): functional_range = create_mock_functional_classification(FunctionalClassificationOptions.not_specified) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == functional_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -104,7 +104,7 @@ def test_variant_not_in_any_range(self): ) score_calibration = create_mock_score_calibration(functional_classifications=[functional_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == ExperimentalVariantFunctionalImpactClassification.INDETERMINATE @@ -122,7 +122,7 @@ def test_multiple_ranges_returns_first_match(self): second_range = create_mock_functional_classification(FunctionalClassificationOptions.abnormal) score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = functional_classification_of_variant(mapped_variant, score_calibration) + result = functional_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == ExperimentalVariantFunctionalImpactClassification.NORMAL @@ -134,7 +134,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -143,7 +143,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -152,7 +152,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - functional_classification_of_variant(mapped_variant, score_calibration) + functional_classification_of_variant(mapped_variant.variant, score_calibration) @pytest.mark.unit @@ -168,7 +168,7 @@ def test_valid_pathogenicity_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -186,7 +186,7 @@ def test_moderate_plus_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) # Verify the mapped evidence strength is MODERATE, not MODERATE_PLUS assert result[0] == pathogenicity_range @@ -201,7 +201,7 @@ def test_none_acmg_classification(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification=None) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -216,7 +216,7 @@ def test_variant_not_in_any_range(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification, variant_in_range=False) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] is None assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -229,7 +229,7 @@ def test_missing_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_empty_score_calibrations_raises_error(self): """Test that empty score calibrations list raises ValueError.""" @@ -238,7 +238,7 @@ def test_empty_score_calibrations_raises_error(self): score_calibration = create_mock_score_calibration() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_missing_functional_classifications_raises_error(self): """Test that missing functional classifications raises ValueError.""" @@ -247,7 +247,7 @@ def test_missing_functional_classifications_raises_error(self): score_calibration = create_mock_score_calibration(functional_classifications=[]) with pytest.raises(ValueError, match="does not have ranges defined in its primary score calibration"): - pathogenicity_classification_of_variant(mapped_variant, score_calibration) + pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) def test_moderate_plus_evidence_strength_maps_to_moderate(self): """Test that MODERATE_PLUS evidence strength maps to Moderate.""" @@ -258,7 +258,7 @@ def test_moderate_plus_evidence_strength_maps_to_moderate(self): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 @@ -282,7 +282,7 @@ def test_various_valid_acmg_combinations(self, criterion, strength): pathogenicity_range = create_mock_pathogenicity_range(acmg_classification) score_calibration = create_mock_score_calibration(functional_classifications=[pathogenicity_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == pathogenicity_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion[criterion] @@ -301,7 +301,7 @@ def test_multiple_ranges_returns_first_match(self): score_calibration = create_mock_score_calibration(functional_classifications=[first_range, second_range]) - result = pathogenicity_classification_of_variant(mapped_variant, score_calibration) + result = pathogenicity_classification_of_variant(mapped_variant.variant, score_calibration) assert result[0] == first_range assert result[1] == VariantPathogenicityEvidenceLine.Criterion.PS3 diff --git a/tests/lib/annotation/test_context.py b/tests/lib/annotation/test_context.py new file mode 100644 index 000000000..e75256512 --- /dev/null +++ b/tests/lib/annotation/test_context.py @@ -0,0 +1,108 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.context — the VA proposition-subject grain (Slice 5.1).""" + +import pytest + +pytest.importorskip("psycopg2") + +from ga4gh.cat_vrs.models import CategoricalVariant +from ga4gh.vrs.models import MolecularVariation + +from mavedb.lib.annotation.context import variant_annotation_context +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +@pytest.mark.integration +class TestVariantAnnotationContextSubject: + """variant_annotation_context — the VA *proposition* subject grain (Slice 5.1). + + The subject always anchors on the **measured** allele. A protein assay unfurls the full equivalence + class of encoders; a nucleotide assay keeps its precise coordinate partner + protein consequence and + excludes the sibling encoders (distinct variants) that the reverse-translation fan left on the record. + Either way the subject is a Cat-VRS ``CategoricalVariant`` when a projection member exists; it falls + back to the concrete measured ``MolecularVariation`` when the categorical can't be assembled. + """ + + def test_protein_assay_subject_is_a_categorical_variant(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="protein", + alleles=[ + AlleleSpec( + digest="prot", level="protein", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="cdna", level="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + + def test_nucleotide_assay_subject_is_a_categorical_variant_over_the_measured_change( + self, session, setup_lib_db_with_mapped_variant + ): + """Anchored on the measured nt change: keep its projection_group coordinate partner + the protein + consequence, exclude the sibling encoder (a distinct variant, different projection group).""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", + level="cdna", + is_authoritative=True, + projection_group=0, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE, + ), + # The measured change's precise coordinate partner (same projection group). + AlleleSpec( + digest="gen", level="genomic", projection_group=0, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + # A sibling encoder of the same protein consequence — distinct variant, other group: excluded. + AlleleSpec( + digest="sibling", level="cdna", projection_group=1, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + AlleleSpec(digest="prot", level="protein", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, CategoricalVariant) + # defining cdna + gen partner + protein apex; the sibling encoder is dropped. + assert len(context.subject_variant.members) == 3 + + def test_single_allele_subject_falls_back_to_molecular_variation(self, session, setup_lib_db_with_mapped_variant): + """A lone measured allele (no projection member) → the categorical build yields the concrete allele.""" + mapped_variant = setup_lib_db_with_mapped_variant + seed_mapping_record( + session, + mapped_variant.variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest="cdna", level="cdna", is_authoritative=True, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE + ), + ], + ) + + context = variant_annotation_context(session, mapped_variant.variant) + + assert context is not None + assert isinstance(context.subject_variant, MolecularVariation) + assert not isinstance(context.subject_variant, CategoricalVariant) + + def test_unmapped_variant_has_no_context(self, session, setup_lib_db_with_mapped_variant): + """No live mapping record on the new substrate → no annotation context.""" + context = variant_annotation_context(session, setup_lib_db_with_mapped_variant.variant) + + assert context is None diff --git a/tests/lib/annotation/test_document.py b/tests/lib/annotation/test_document.py index 0a9b0ac86..d87e9b227 100644 --- a/tests/lib/annotation/test_document.py +++ b/tests/lib/annotation/test_document.py @@ -8,6 +8,7 @@ """ import urllib.parse +from types import SimpleNamespace import pytest @@ -19,8 +20,7 @@ from mavedb.lib.annotation.document import ( experiment_as_iri, experiment_to_document, - mapped_variant_as_iri, - mapped_variant_to_document, + measured_allele_as_iri, score_calibration_as_document, score_set_as_iri, score_set_to_document, @@ -81,42 +81,24 @@ def test_score_set_to_document(self, mock_score_set): @pytest.mark.unit -class TestMappedVariantDocumentFunctions: - """Unit tests for mapped variant document creation functions.""" +class TestMeasuredAlleleDocumentFunctions: + """Unit tests for measured-allele IRI generation.""" - def test_mapped_variant_as_iri(self, mock_mapped_variant): - """Test IRI generation for mapped variants with ClinGen allele ID.""" - expected_iri_root = ( - f"https://mavedb.org/variant/{urllib.parse.quote_plus(mock_mapped_variant.clingen_allele_id)}" - ) - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri(self): + """Test IRI generation for a measured allele with a ClinGen allele ID.""" + allele = SimpleNamespace(clingen_allele_id="CA123456") + expected_iri_root = f"https://mavedb.org/variant/{urllib.parse.quote_plus(allele.clingen_allele_id)}" + result = measured_allele_as_iri(allele) assert result.root == expected_iri_root - def test_mapped_variant_as_iri_no_caid(self, mock_mapped_variant): - """Test IRI generation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_as_iri(mock_mapped_variant) + def test_measured_allele_as_iri_no_caid(self): + """Test IRI generation for a measured allele without a ClinGen allele ID returns None.""" + allele = SimpleNamespace(clingen_allele_id=None) + result = measured_allele_as_iri(allele) assert result is None - def test_mapped_variant_to_document(self, mock_mapped_variant): - """Test document creation for mapped variants with ClinGen allele ID.""" - document = mapped_variant_to_document(mock_mapped_variant) - - assert document.id == mock_mapped_variant.variant.urn - assert document.name == "MaveDB Mapped Variant" - assert document.documentType == "mapped genomic variant description" - assert len(document.urls) > 0 - assert mapped_variant_as_iri(mock_mapped_variant).root in document.urls - - def test_mapped_variant_to_document_no_caid(self, mock_mapped_variant): - """Test document creation for mapped variants without ClinGen allele ID returns None.""" - mock_mapped_variant.clingen_allele_id = None - document = mapped_variant_to_document(mock_mapped_variant) - - assert document is None - @pytest.mark.unit class TestVariantDocumentFunctions: diff --git a/tests/lib/annotation/test_eligibility.py b/tests/lib/annotation/test_eligibility.py new file mode 100644 index 000000000..239ed5af3 --- /dev/null +++ b/tests/lib/annotation/test_eligibility.py @@ -0,0 +1,375 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.annotation.eligibility — variant annotatability predicates.""" + +from copy import deepcopy +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.annotation.eligibility import ( + _can_annotate_variant_base_assumptions, + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, + can_annotate_variant_for_functional_statement, + can_annotate_variant_for_pathogenicity_evidence, +) + + +@pytest.mark.unit +class TestBaseAnnotationAssumptionsUnit: + def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): + mock_mapped_variant.variant.data = {"score_data": {"score": None}} + + assert _can_annotate_variant_base_assumptions(mock_mapped_variant.variant) is False + + def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): + assert _can_annotate_variant_base_assumptions(mock_mapped_variant.variant) is True + + +@pytest.mark.unit +class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: + """ + Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. + This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. + """ + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): + mock_mapped_variant.variant.score_set.score_calibrations = None + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): + mock_mapped_variant.variant.score_set.score_calibrations = [] + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) + def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( + self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type + ): + """Test that research use only calibrations are excluded by default.""" + # Make all calibrations research use only + for ( + calibration + ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: + calibration.research_use_only = True + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, annotation_type + ) + is False + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( + self, kind, variant_fixture, request + ): + """Test that research use only calibrations are included when explicitly allowed.""" + mock_mapped_variant = request.getfixturevalue(variant_fixture) + # Make all calibrations research use only + for calibration in mock_mapped_variant.variant.score_set.score_calibrations: + calibration.primary = False + calibration.research_use_only = True + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind, allow_research_use_only_calibrations=True + ) + is True + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( + self, kind, variant_fixture, request + ): + mock_mapped_variant = request.getfixturevalue(variant_fixture) + + for calibration in mock_mapped_variant.variant.score_set.score_calibrations: + calibration.functional_classifications = None + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is False + ) + + def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + for fr in acmg_classification_removed: + fr["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is False + ) + + def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + for ( + calibration + ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: + acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] + acmg_classification_removed[0]["acmgClassification"] = None + + calibration.functional_classifications = acmg_classification_removed + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is True + ) + + @pytest.mark.parametrize( + "kind,variant_fixture", + [ + ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), + ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), + ], + ids=["functional_fixture", "pathogenicity_fixture"], + ) + def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( + self, kind, variant_fixture, request + ): + mock_mapped_variant = request.getfixturevalue(variant_fixture) + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant.variant, kind + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for functional annotation.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + # Create a copy of the existing calibration + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # Make the first one research use only, leave the second as regular + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + # Should return True because at least one non-research-only calibration has valid classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is True + ) + + def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" + calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + # Create a copy of the existing calibration + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # Make the first one research use only, leave the second as regular + calibrations[0].research_use_only = True + calibrations[1].research_use_only = False + + # Should return True because at least one non-research-only calibration has valid classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant, "pathogenicity" + ) + is True + ) + + def test_score_range_check_handles_mixed_functional_classifications( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test behavior when some calibrations have functional classifications and some don't.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # If there's only one calibration, add another for testing + if len(calibrations) == 1: + new_calibration = deepcopy(calibrations[0]) + calibrations.append(new_calibration) + + # First calibration has functional classifications (should already exist) + # Second calibration has no functional classifications + calibrations[1].functional_classifications = None + + # Should return True because at least one calibration has valid functional classifications + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is True + ) + + def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # Remove ACMG classifications from all functional classifications + for calibration in calibrations: + if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: + acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] + for fc in acmg_classification_removed: + if "acmgClassification" in fc: + fc["acmgClassification"] = None + calibration.functional_classifications = acmg_classification_removed + + # Should return False because no ACMG classifications exist + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "pathogenicity" + ) + is False + ) + + def test_functional_annotation_with_empty_functional_classifications_list( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + """Test that functional annotation fails when functional classifications list is empty.""" + calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations + + # Set functional classifications to empty list + for calibration in calibrations: + calibration.functional_classifications = [] + + assert ( + _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + mock_mapped_variant_with_functional_calibration_score_set.variant, "functional" + ) + is False + ) + + +@pytest.mark.unit +class TestPathogenicityAnnotationEligibilityUnit: + def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", return_value=False): + result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant.variant) + + assert result is False + + def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): + with patch( + "mavedb.lib.annotation.eligibility._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", + return_value=False, + ): + result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant.variant) + + assert result is False + + def test_pathogenicity_range_check_returns_true_when_all_conditions_met( + self, + mock_mapped_variant_with_pathogenicity_calibration_score_set, + ): + assert ( + can_annotate_variant_for_pathogenicity_evidence( + mock_mapped_variant_with_pathogenicity_calibration_score_set.variant + ) + is True + ) + + +@pytest.mark.unit +class TestFunctionalAnnotationEligibilityUnit: + def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): + with patch( + "mavedb.lib.annotation.eligibility._can_annotate_variant_base_assumptions", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_mapped_variant.variant) + + assert result is False + + def test_functional_range_check_returns_false_when_functional_classifications_check_fails( + self, mock_mapped_variant + ): + with patch( + "mavedb.lib.annotation.eligibility._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", + return_value=False, + ): + result = can_annotate_variant_for_functional_statement(mock_mapped_variant.variant) + + assert result is False + + def test_functional_range_check_returns_true_when_all_conditions_met( + self, + mock_mapped_variant_with_functional_calibration_score_set, + ): + assert ( + can_annotate_variant_for_functional_statement( + mock_mapped_variant_with_functional_calibration_score_set.variant + ) + is True + ) + + +@pytest.mark.integration +class TestAnnotationEligibilityIntegration: + def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): + # Make score presence explicit so a negative result is due to missing calibrations. + setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} + + pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence( + setup_lib_db_with_mapped_variant.variant + ) + functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant.variant) + + # DB fixture score sets do not include calibrations by default, so both should be False. + assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] + assert pathogenicity_allowed is False + assert functional_allowed is False diff --git a/tests/lib/annotation/test_evidence_line.py b/tests/lib/annotation/test_evidence_line.py index fb6f33b74..640b19a5d 100644 --- a/tests/lib/annotation/test_evidence_line.py +++ b/tests/lib/annotation/test_evidence_line.py @@ -21,10 +21,10 @@ from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) -from mavedb.lib.annotation.statement import mapped_variant_to_functional_statement +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit @@ -49,23 +49,23 @@ class TestAcmgEvidenceLine: ) def test_acmg_evidence_line_with_met_valid_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, expected_outcome, expected_strength, expected_direction, ): """Test ACMG evidence line creation with met valid clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) if expected_strength == StrengthOfEvidenceProvided.STRONG: expected_evidence_outcome = expected_outcome.value @@ -73,7 +73,7 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( expected_evidence_outcome = f"{expected_outcome.value}_{expected_strength.name.lower()}" assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Met" @@ -87,11 +87,11 @@ def test_acmg_evidence_line_with_met_valid_clinical_classification( def test_acmg_evidence_line_with_not_met_clinical_classification( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Test ACMG evidence line creation with not met clinical classification.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] expected_outcome = VariantPathogenicityEvidenceLine.Criterion.PS3 expected_strength = None expected_evidence_outcome = f"{expected_outcome.value}_not_met" @@ -100,13 +100,13 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( "mavedb.lib.annotation.evidence_line.pathogenicity_classification_of_variant", return_value=(MagicMock(label="Test Range"), expected_outcome, expected_strength), ): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - study_result = variant_study_result(mapped_variant) - evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - result = acmg_evidence_line(mapped_variant, score_calibration, proposition, [evidence]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + evidence = functional_evidence_line(context, score_calibration, [study_result]) + result = acmg_evidence_line(context, score_calibration, proposition, [evidence]) assert isinstance(result, VariantPathogenicityEvidenceLine) - assert result.description == f"Pathogenicity evidence line for {mapped_variant.variant.urn}." + assert result.description == f"Pathogenicity evidence line for {context.variant.urn}." assert result.evidenceOutcome.primaryCoding.code.root == expected_evidence_outcome assert result.evidenceOutcome.primaryCoding.system == "ACMG Guidelines, 2015" assert result.evidenceOutcome.name == f"ACMG 2015 {expected_outcome.name} Criterion Not Met" @@ -117,19 +117,20 @@ def test_acmg_evidence_line_with_not_met_clinical_classification( assert result.targetProposition == proposition assert len(result.hasEvidenceItems) == 1 - def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_mapped_variant): + def test_acmg_evidence_line_with_no_calibrations_raises_error(self, mock_annotation_context): """Test that ACMG evidence line creation raises error when no calibrations exist.""" - mock_mapped_variant.variant.score_set.score_calibrations = None + context = mock_annotation_context + context.variant.score_set.score_calibrations = None score_calibration = MagicMock() with pytest.raises(ValueError, match="does not have a score set with score calibrations"): - proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) - study_result = variant_study_result(mock_mapped_variant) - acmg_evidence_line(mock_mapped_variant, score_calibration, proposition, [study_result]) + proposition = variant_pathogenicity_proposition(context) + study_result = variant_study_result(context) + acmg_evidence_line(context, score_calibration, proposition, [study_result]) def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_error( self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, + mock_annotation_context_with_pathogenicity_calibration_score_set, ): """Regression test: VariantPathogenicityEvidenceLine must accept Statement instances directly. @@ -138,22 +139,22 @@ def test_acmg_evidence_line_accepts_statement_evidence_without_serialization_err nested VRS objects from those dicts (e.g. Allele with real genomic coordinates). Evidence model instances must be passed directly, not serialized. """ - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - - study_result = variant_study_result(mapped_variant) - functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - functional_evidence = functional_evidence_line(mapped_variant, score_calibration, [study_result]) - functional_statement = mapped_variant_to_functional_statement( - mapped_variant, + context = mock_annotation_context_with_pathogenicity_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + + study_result = variant_study_result(context) + functional_proposition = variant_functional_impact_proposition(context) + functional_evidence = functional_evidence_line(context, score_calibration, [study_result]) + statement = functional_statement( + context, functional_proposition, [functional_evidence], score_calibration, ExperimentalVariantFunctionalImpactClassification.NORMAL, ) - clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) + clinical_proposition = variant_pathogenicity_proposition(context) - result = acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) + result = acmg_evidence_line(context, score_calibration, clinical_proposition, [statement]) assert isinstance(result, VariantPathogenicityEvidenceLine) assert len(result.hasEvidenceItems) == 1 @@ -166,16 +167,16 @@ class TestFunctionalEvidenceLine: """Unit tests for functional evidence line creation.""" def test_functional_evidence_line_with_valid_functional_evidence( - self, mock_mapped_variant_with_functional_calibration_score_set + self, mock_annotation_context_with_functional_calibration_score_set ): """Test functional evidence line creation with valid evidence.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] - study_result = variant_study_result(mapped_variant) - result = functional_evidence_line(mapped_variant, score_calibration, [study_result]) + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] + study_result = variant_study_result(context) + result = functional_evidence_line(context, score_calibration, [study_result]) assert isinstance(result, EvidenceLine) - assert result.description == f"Functional evidence line for {mapped_variant.variant.urn}" + assert result.description == f"Functional evidence line for {context.variant.urn}" assert result.directionOfEvidenceProvided is not None assert result.specifiedBy assert result.contributions diff --git a/tests/lib/annotation/test_proposition.py b/tests/lib/annotation/test_proposition.py index cb883e0bc..b7c905c60 100644 --- a/tests/lib/annotation/test_proposition.py +++ b/tests/lib/annotation/test_proposition.py @@ -7,51 +7,164 @@ clinical and functional impact propositions. """ +from types import SimpleNamespace +from unittest.mock import patch + import pytest pytest.importorskip("psycopg2") +from ga4gh.cat_vrs.models import CategoricalVariant from ga4gh.va_spec.base.core import ( ExperimentalVariantFunctionalImpactProposition, VariantPathogenicityProposition, ) from ga4gh.vrs.models import MolecularVariation +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( - mapped_variant_to_experimental_variant_clinical_impact_proposition, - mapped_variant_to_experimental_variant_functional_impact_proposition, + sequence_feature_for_variant, + variant_functional_impact_proposition, + variant_pathogenicity_proposition, ) +from mavedb.lib.cat_vrs import build_categorical_variant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record_allele import MappingRecordAllele +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE +from tests.lib.annotation.conftest import annotation_context_for + + +def _protein_categorical_variant() -> CategoricalVariant: + """A Cat-VRS ``CategoricalVariant`` for a protein-measured variant (defining protein + `encodes` nt). + + Built over transient links (no DB) so the proposition tests can assert the builders accept a + categorical subject — the shape a protein assay produces under Slice 5.1. + """ + links = [ + MappingRecordAllele( + is_authoritative=True, + allele=Allele(level="protein", vrs_digest="prot", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + MappingRecordAllele( + is_authoritative=False, + allele=Allele(level="cdna", vrs_digest="cdna", post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE), + ), + ] + transit = build_categorical_variant(links, name="urn:mavedb:test#1") + assert transit is not None + return transit.categorical_variant @pytest.mark.unit class TestExperimentalVariantClinicalImpactProposition: """Unit tests for experimental variant clinical impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_clinical_impact_proposition(self, mock_mapped_variant): - """Test creation of clinical impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_clinical_impact_proposition(mock_mapped_variant) + def test_variant_pathogenicity_proposition(self, mock_annotation_context): + """Test creation of pathogenicity proposition from a variant annotation context.""" + result = variant_pathogenicity_proposition(mock_annotation_context) assert isinstance(result, VariantPathogenicityProposition) - assert result.description == f"Variant pathogenicity proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant pathogenicity proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "isCausalFor" assert result.objectCondition.root.conceptType == "Disease" assert result.objectCondition.root.primaryCoding.code.root == "C0012634" assert result.objectCondition.root.primaryCoding.system == "https://www.ncbi.nlm.nih.gov/medgen/" + def test_clinical_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_pathogenicity_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + @pytest.mark.unit class TestExperimentalVariantFunctionalImpactProposition: """Unit tests for experimental variant functional impact proposition creation.""" - def test_mapped_variant_to_experimental_variant_functional_impact_proposition(self, mock_mapped_variant): - """Test creation of functional impact proposition from mapped variant.""" - result = mapped_variant_to_experimental_variant_functional_impact_proposition(mock_mapped_variant) + def test_variant_functional_impact_proposition(self, mock_annotation_context): + """Test creation of functional impact proposition from a variant annotation context.""" + result = variant_functional_impact_proposition(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactProposition) - assert result.description == f"Variant functional impact proposition for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact proposition for {mock_annotation_context.variant.urn}." assert isinstance(result.subjectVariant, MolecularVariation) assert result.predicate == "impactsFunctionOf" assert result.objectSequenceFeature.primaryCoding.code.root == "BRCA1" assert result.objectSequenceFeature.primaryCoding.system == "https://www.genenames.org/" assert result.experimentalContextQualifier is not None + + def test_functional_impact_proposition_carries_a_categorical_subject(self, mock_mapped_variant): + """A protein assay's categorical subject (Slice 5.1) is carried onto the proposition unchanged.""" + context = annotation_context_for(mock_mapped_variant, subject_variant=_protein_categorical_variant()) + result = variant_functional_impact_proposition(context) + + assert isinstance(result.subjectVariant, CategoricalVariant) + + +@pytest.mark.unit +class TestSequenceFeatureForVariantUnit: + """sequence_feature_for_variant — co-located with the propositions that consume it.""" + + def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=None): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) + + def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["ENST00000357654"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "ENST00000357654" + assert system == "https://www.ensembl.org/index.html" + + def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["NM_000546.6"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "NM_000546.6" + assert system == "https://www.ncbi.nlm.nih.gov/refseq/" + + def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch( + "mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", + return_value=["CUSTOM_ID_1"], + ): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "CUSTOM_ID_1" + assert system == "transcript or gene identifier of unknown source" + + def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + feature, system = sequence_feature_for_variant(mock_mapped_variant.variant) + + assert feature == "TP53" + assert system == "https://www.mavedb.org/" + + def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): + target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) + + with patch("mavedb.lib.annotation.proposition.target_for_variant", return_value=target): + with patch("mavedb.lib.annotation.proposition.extract_ids_from_post_mapped_metadata", return_value=[]): + with pytest.raises(MappingDataDoesntExistException): + sequence_feature_for_variant(mock_mapped_variant.variant) diff --git a/tests/lib/annotation/test_statement.py b/tests/lib/annotation/test_statement.py index 5c36aeecf..7cd7e41fb 100644 --- a/tests/lib/annotation/test_statement.py +++ b/tests/lib/annotation/test_statement.py @@ -3,7 +3,7 @@ """ Tests for mavedb.lib.annotation.statement module. -This module tests statement creation functions for mapped variants, +This module tests statement creation functions for variants, focusing on functional impact statements and their classification. """ @@ -18,15 +18,13 @@ from mavedb.lib.annotation.annotate import variant_study_result from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification from mavedb.lib.annotation.evidence_line import functional_evidence_line -from mavedb.lib.annotation.proposition import mapped_variant_to_experimental_variant_functional_impact_proposition -from mavedb.lib.annotation.statement import ( - mapped_variant_to_functional_statement, -) +from mavedb.lib.annotation.proposition import variant_functional_impact_proposition +from mavedb.lib.annotation.statement import functional_statement @pytest.mark.unit -class TestMappedVariantFunctionalStatement: - """Unit tests for mapped variant functional statement creation.""" +class TestFunctionalStatement: + """Unit tests for functional statement creation.""" @pytest.mark.parametrize( "classification, expected_direction", @@ -36,30 +34,26 @@ class TestMappedVariantFunctionalStatement: (ExperimentalVariantFunctionalImpactClassification.INDETERMINATE, Direction.NEUTRAL), ], ) - def test_mapped_variant_to_functional_statement( + def test_functional_statement( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, classification, expected_direction, ): """Test functional statement creation with different classifications.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] with patch( "mavedb.lib.annotation.evidence_line.functional_classification_of_variant", return_value=(MagicMock(label="Test Range"), classification), ): - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - result = mapped_variant_to_functional_statement( - mapped_variant, proposition, [evidence], score_calibration, classification - ) + proposition = variant_functional_impact_proposition(context) + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + result = functional_statement(context, proposition, [evidence], score_calibration, classification) assert isinstance(result, Statement) - assert result.description == f"Variant functional impact statement for {mapped_variant.variant.urn}." + assert result.description == f"Variant functional impact statement for {context.variant.urn}." assert result.specifiedBy assert result.contributions assert len(result.contributions) == 3 # API, VRS, and score calibration contributions @@ -75,24 +69,22 @@ def test_mapped_variant_to_functional_statement( def test_no_calibrations_raises_value_error( self, - mock_mapped_variant_with_functional_calibration_score_set, + mock_annotation_context_with_functional_calibration_score_set, ): """Test that missing score calibrations raises ValueError in evidence line creation.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - score_calibration = mapped_variant.variant.score_set.score_calibrations[0] + context = mock_annotation_context_with_functional_calibration_score_set + score_calibration = context.variant.score_set.score_calibrations[0] # Set calibrations to None to trigger the error - mapped_variant.variant.score_set.score_calibrations = None + context.variant.score_set.score_calibrations = None - proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) + proposition = variant_functional_impact_proposition(context) with pytest.raises(ValueError, match="does not have a score set with score calibrations"): # Use a dummy score calibration for evidence line creation, but it should not be reached due to the error - evidence = functional_evidence_line( - mapped_variant, score_calibration, [variant_study_result(mapped_variant)] - ) - mapped_variant_to_functional_statement( - mapped_variant, + evidence = functional_evidence_line(context, score_calibration, [variant_study_result(context)]) + functional_statement( + context, proposition, [evidence], score_calibration, @@ -101,8 +93,8 @@ def test_no_calibrations_raises_value_error( @pytest.mark.unit -class TestMappedVariantPathogenicityStatement: - """Unit tests for mapped variant pathogenicity statement ACMG classification mapping.""" +class TestPathogenicityStatement: + """Unit tests for pathogenicity statement ACMG classification mapping.""" def test_pathogenic_criterion_maps_to_pathogenic(self, mock_mapped_variant): """Test that pathogenic ACMG criterion (PS3) maps to PATHOGENIC classification.""" diff --git a/tests/lib/annotation/test_study_result.py b/tests/lib/annotation/test_study_result.py index 1cf8bbf63..356eaa1e2 100644 --- a/tests/lib/annotation/test_study_result.py +++ b/tests/lib/annotation/test_study_result.py @@ -14,24 +14,22 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult from ga4gh.vrs.models import MolecularVariation -from mavedb.lib.annotation.document import mapped_variant_as_iri, variant_as_iri -from mavedb.lib.annotation.study_result import ( - mapped_variant_to_experimental_variant_impact_study_result, -) +from mavedb.lib.annotation.document import measured_allele_as_iri, variant_as_iri +from mavedb.lib.annotation.study_result import variant_impact_study_result @pytest.mark.unit class TestExperimentalVariantImpactStudyResult: """Unit tests for experimental variant impact study result creation.""" - def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_mapped_variant): - """Test creation of experimental variant impact study result from mapped variant.""" - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_variant_impact_study_result(self, mock_annotation_context): + """Test creation of experimental variant impact study result from an annotation context.""" + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -39,18 +37,18 @@ def test_mapped_variant_to_experimental_variant_impact_study_result(self, mock_m # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert mapped_variant_as_iri(mock_mapped_variant) in result.reportedIn - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert measured_allele_as_iri(mock_annotation_context.measured_allele) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn - def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): - """Test that study result handles missing mapped variant (no ClinGen allele ID).""" - mock_mapped_variant.clingen_allele_id = None - result = mapped_variant_to_experimental_variant_impact_study_result(mock_mapped_variant) + def test_no_clingen_allele_id_is_filtered_properly(self, mock_annotation_context): + """Test that study result handles a measured allele with no ClinGen allele ID.""" + mock_annotation_context.measured_allele.clingen_allele_id = None + result = variant_impact_study_result(mock_annotation_context) assert isinstance(result, ExperimentalVariantFunctionalImpactStudyResult) - assert result.description == f"Variant effect study result for {mock_mapped_variant.variant.urn}." + assert result.description == f"Variant effect study result for {mock_annotation_context.variant.urn}." assert isinstance(result.focusVariant, MolecularVariation) - assert result.functionalImpactScore == mock_mapped_variant.variant.data["score_data"]["score"] + assert result.functionalImpactScore == mock_annotation_context.variant.data["score_data"]["score"] # Verify all expected contribution types are present contribution_types = {c.name for c in result.contributions} expected_types = {"MaveDB API", "MaveDB VRS Mapper", "MaveDB Dataset Creator", "MaveDB Dataset Modifier"} @@ -58,5 +56,5 @@ def test_no_mapped_variant_is_filtered_properly(self, mock_mapped_variant): # specifiedBy will be None when no primary publications exist assert result.sourceDataSet is not None assert result.reportedIn is not None - assert variant_as_iri(mock_mapped_variant.variant) in result.reportedIn + assert variant_as_iri(mock_annotation_context.variant) in result.reportedIn assert len(result.reportedIn) == 1 diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py deleted file mode 100644 index 515ae6286..000000000 --- a/tests/lib/annotation/test_util.py +++ /dev/null @@ -1,868 +0,0 @@ -# ruff: noqa: E402 - -""" -Tests for mavedb.lib.annotation.util module. - -This module tests utility functions used by annotation workflows, including -variation extraction, annotation eligibility checks, and sequence feature -resolution. -""" - -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -pytest.importorskip("psycopg2") - -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.annotation.util import ( - _can_annotate_variant_base_assumptions, - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, - can_annotate_variant_for_functional_statement, - can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, - select_strongest_functional_calibration, - select_strongest_pathogenicity_calibration, - sequence_feature_for_mapped_variant, - variation_from_mapped_variant, - vrs_object_from_mapped_variant, -) -from tests.helpers.constants import ( - TEST_SEQUENCE_LOCATION_ACCESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE, - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, -) - - -@pytest.mark.unit -class TestVariationExtractionUnit: - @pytest.mark.parametrize( - "variation_version", - [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], - ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], - ) - def test_variation_from_mapped_variant_post_mapped_variation(self, mock_mapped_variant, variation_version): - mock_mapped_variant.post_mapped = variation_version - - result = variation_from_mapped_variant(mock_mapped_variant).model_dump() - - assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION - assert result["location"]["start"] == 5 - assert result["location"]["end"] == 6 - - def test_variation_from_mapped_variant_no_post_mapped(self, mock_mapped_variant): - mock_mapped_variant.post_mapped = None - - with pytest.raises(MappingDataDoesntExistException): - variation_from_mapped_variant(mock_mapped_variant) - - @pytest.mark.parametrize( - "allele_dict, expected_state_type, expected_state_fields", - [ - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE, - "LiteralSequenceExpression", - {"sequence": "F"}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, - "ReferenceLengthExpression", - {"length": 5, "repeatSubunitLength": 5}, - ), - ( - TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, - "LengthExpression", - {"length": 5}, - ), - ], - ids=["lse_state", "rle_state", "length_expression_state"], - ) - def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): - result = vrs_object_from_mapped_variant(allele_dict).model_dump() - - assert result["state"]["type"] == expected_state_type - for k, v in expected_state_fields.items(): - assert result["state"][k] == v - - def test_unknown_allele_state_type_raises_value_error(self): - allele_dict = { - **TEST_VALID_POST_MAPPED_VRS_ALLELE, - "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, - } - - with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): - vrs_object_from_mapped_variant(allele_dict) - - @pytest.mark.parametrize( - "member_dict, expected_state_type", - [ - (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), - (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), - ], - ids=["lse_members", "rle_members", "length_expression_members"], - ) - def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): - mapping_results = {"type": "CisPhasedBlock", "members": [member_dict, member_dict]} - - result = vrs_object_from_mapped_variant(mapping_results).model_dump() - - assert result["type"] == "CisPhasedBlock" - assert len(result["members"]) == 2 - assert result["members"][0]["state"]["type"] == expected_state_type - - -@pytest.mark.unit -class TestBaseAnnotationAssumptionsUnit: - def test_base_assumption_check_returns_false_when_score_is_none(self, mock_mapped_variant): - mock_mapped_variant.variant.data = {"score_data": {"score": None}} - - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is False - - def test_base_assumption_check_returns_true_when_all_conditions_met(self, mock_mapped_variant): - assert _can_annotate_variant_base_assumptions(mock_mapped_variant) is True - - -@pytest.mark.unit -class TestScoreCalibrationMayBeUsedForAnnotation: - def test_returns_false_for_research_use_only_when_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=False, - ) - is False - ) - - def test_returns_true_for_research_use_only_when_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.research_use_only = True - - assert ( - score_calibration_may_be_used_for_annotation( - calibration, - annotation_type="functional", - allow_research_use_only_calibrations=True, - ) - is True - ) - - def test_returns_false_when_functional_classifications_missing( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - calibration = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations[0] - calibration.functional_classifications = [] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="functional") is False - - def test_returns_false_for_pathogenicity_without_acmg_classifications( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - acmg_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for functional_classification in acmg_removed: - functional_classification["acmgClassification"] = None - calibration.functional_classifications = acmg_removed - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is False - - def test_returns_true_for_pathogenicity_with_any_acmg_classification( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - calibration = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations[ - 0 - ] - - assert score_calibration_may_be_used_for_annotation(calibration, annotation_type="pathogenicity") is True - - -@pytest.mark.unit -class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: - """ - Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. - This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. - """ - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): - mock_mapped_variant.variant.score_set.score_calibrations = [] - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) - def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( - self, mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ): - """Test that research use only calibrations are excluded by default.""" - # Make all calibrations research use only - for ( - calibration - ) in mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations: - calibration.research_use_only = True - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ) - is False - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_research_use_only_calibrations_are_allowed( - self, kind, variant_fixture, request - ): - """Test that research use only calibrations are included when explicitly allowed.""" - mock_mapped_variant = request.getfixturevalue(variant_fixture) - # Make all calibrations research use only - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.primary = False - calibration.research_use_only = True - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant, kind, allow_research_use_only_calibrations=True - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_false_when_calibrations_present_with_empty_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - for calibration in mock_mapped_variant.variant.score_set.score_calibrations: - calibration.functional_classifications = None - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) - - def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - for fr in acmg_classification_removed: - fr["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - for ( - calibration - ) in mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations: - acmg_classification_removed = [deepcopy(r) for r in calibration.functional_classifications] - acmg_classification_removed[0]["acmgClassification"] = None - - calibration.functional_classifications = acmg_classification_removed - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - @pytest.mark.parametrize( - "kind,variant_fixture", - [ - ("functional", "mock_mapped_variant_with_functional_calibration_score_set"), - ("pathogenicity", "mock_mapped_variant_with_pathogenicity_calibration_score_set"), - ], - ids=["functional_fixture", "pathogenicity_fixture"], - ) - def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges( - self, kind, variant_fixture, request - ): - mock_mapped_variant = request.getfixturevalue(variant_fixture) - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for functional annotation.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is True - ) - - def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_pathogenicity( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test behavior with mixed research use only and regular calibrations for pathogenicity annotation.""" - calibrations = mock_mapped_variant_with_pathogenicity_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - # Create a copy of the existing calibration - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # Make the first one research use only, leave the second as regular - calibrations[0].research_use_only = True - calibrations[1].research_use_only = False - - # Should return True because at least one non-research-only calibration has valid classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" - ) - is True - ) - - def test_score_range_check_handles_mixed_functional_classifications( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test behavior when some calibrations have functional classifications and some don't.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # If there's only one calibration, add another for testing - if len(calibrations) == 1: - new_calibration = deepcopy(calibrations[0]) - calibrations.append(new_calibration) - - # First calibration has functional classifications (should already exist) - # Second calibration has no functional classifications - calibrations[1].functional_classifications = None - - # Should return True because at least one calibration has valid functional classifications - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is True - ) - - def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that pathogenicity annotation fails when functional classifications exist but have no ACMG classifications.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Remove ACMG classifications from all functional classifications - for calibration in calibrations: - if hasattr(calibration, "functional_classifications") and calibration.functional_classifications: - acmg_classification_removed = [deepcopy(fc) for fc in calibration.functional_classifications] - for fc in acmg_classification_removed: - if "acmgClassification" in fc: - fc["acmgClassification"] = None - calibration.functional_classifications = acmg_classification_removed - - # Should return False because no ACMG classifications exist - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity" - ) - is False - ) - - def test_functional_annotation_with_empty_functional_classifications_list( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - """Test that functional annotation fails when functional classifications list is empty.""" - calibrations = mock_mapped_variant_with_functional_calibration_score_set.variant.score_set.score_calibrations - - # Set functional classifications to empty list - for calibration in calibrations: - calibration.functional_classifications = [] - - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) - is False - ) - - -@pytest.mark.unit -class TestPathogenicityAnnotationEligibilityUnit: - def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", return_value=False): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, - ): - result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) - - assert result is False - - def test_pathogenicity_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_pathogenicity_calibration_score_set, - ): - assert ( - can_annotate_variant_for_pathogenicity_evidence( - mock_mapped_variant_with_pathogenicity_calibration_score_set - ) - is True - ) - - -@pytest.mark.unit -class TestFunctionalAnnotationEligibilityUnit: - def test_functional_range_check_returns_false_when_base_assumptions_fail(self, mock_mapped_variant): - with patch( - "mavedb.lib.annotation.util._can_annotate_variant_base_assumptions", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_false_when_functional_classifications_check_fails( - self, mock_mapped_variant - ): - with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, - ): - result = can_annotate_variant_for_functional_statement(mock_mapped_variant) - - assert result is False - - def test_functional_range_check_returns_true_when_all_conditions_met( - self, - mock_mapped_variant_with_functional_calibration_score_set, - ): - assert ( - can_annotate_variant_for_functional_statement(mock_mapped_variant_with_functional_calibration_score_set) - is True - ) - - -@pytest.mark.unit -class TestSequenceFeatureForMappedVariantUnit: - def test_sequence_feature_raises_when_target_is_missing(self, mock_mapped_variant): - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=None): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - def test_sequence_feature_returns_ensembl_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["ENST00000357654"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "ENST00000357654" - assert system == "https://www.ensembl.org/index.html" - - def test_sequence_feature_returns_refseq_identifier(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["NM_000546.6"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "NM_000546.6" - assert system == "https://www.ncbi.nlm.nih.gov/refseq/" - - def test_sequence_feature_returns_unknown_identifier_source(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={"x": "y"}, name="BRCA1") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch( - "mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=["CUSTOM_ID_1"] - ): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "CUSTOM_ID_1" - assert system == "transcript or gene identifier of unknown source" - - def test_sequence_feature_falls_back_to_target_name(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name="TP53") - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - feature, system = sequence_feature_for_mapped_variant(mock_mapped_variant) - - assert feature == "TP53" - assert system == "https://www.mavedb.org/" - - def test_sequence_feature_raises_when_target_has_no_name_or_ids(self, mock_mapped_variant): - target = SimpleNamespace(mapped_hgnc_name=None, post_mapped_metadata={}, name=None) - - with patch("mavedb.lib.annotation.util.target_for_variant", return_value=target): - with patch("mavedb.lib.annotation.util.extract_ids_from_post_mapped_metadata", return_value=[]): - with pytest.raises(MappingDataDoesntExistException): - sequence_feature_for_mapped_variant(mock_mapped_variant) - - -@pytest.mark.integration -class TestAnnotationUtilIntegration: - """Integration tests for utility helpers using persisted DB-backed variants.""" - - def test_variation_from_persisted_mapped_variant(self, setup_lib_db_with_mapped_variant): - setup_lib_db_with_mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE - variation = variation_from_mapped_variant(setup_lib_db_with_mapped_variant) - - assert variation is not None - assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} - - def test_annotation_eligibility_returns_boolean_for_persisted_variant(self, setup_lib_db_with_mapped_variant): - # Make score presence explicit so a negative result is due to missing calibrations. - setup_lib_db_with_mapped_variant.variant.data = {"score_data": {"score": 1.0}} - - pathogenicity_allowed = can_annotate_variant_for_pathogenicity_evidence(setup_lib_db_with_mapped_variant) - functional_allowed = can_annotate_variant_for_functional_statement(setup_lib_db_with_mapped_variant) - - # DB fixture score sets do not include calibrations by default, so both should be False. - assert setup_lib_db_with_mapped_variant.variant.score_set.score_calibrations == [] - assert pathogenicity_allowed is False - assert functional_allowed is False - - -@pytest.mark.unit -class TestSelectStrongestFunctionalCalibrationUnit: - """Unit tests for select_strongest_functional_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_functional_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_returns_first_when_all_agree(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that first calibration is returned when all have same classification.""" - from copy import deepcopy - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - # Create multiple calibrations with same classification - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] # Should return the first one - - def test_defaults_to_normal_on_conflict(self, mock_mapped_variant_with_functional_calibration_score_set): - """Test that normal classification is preferred when there are conflicts.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different classifications - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - side_effect=[ - (MagicMock(label="Abnormal"), ExperimentalVariantFunctionalImpactClassification.ABNORMAL), - (MagicMock(label="Normal"), ExperimentalVariantFunctionalImpactClassification.NORMAL), - ], - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return the normal classification (second one) - assert calibration == calibration2 - assert functional_range.label == "Normal" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_functional_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from mavedb.lib.annotation.classification import ExperimentalVariantFunctionalImpactClassification - - mapped_variant = mock_mapped_variant_with_functional_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range but INDETERMINATE classification (variant not in any range) - with patch( - "mavedb.lib.annotation.util.functional_classification_of_variant", - return_value=(None, ExperimentalVariantFunctionalImpactClassification.INDETERMINATE), - ): - calibration, functional_range = select_strongest_functional_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None - - -@pytest.mark.unit -class TestSelectStrongestPathogenicityCalibrationUnit: - """Unit tests for select_strongest_pathogenicity_calibration function.""" - - def test_returns_none_for_empty_calibrations(self, mock_mapped_variant): - """Test that empty calibration list returns None.""" - calibration, functional_range = select_strongest_pathogenicity_calibration(mock_mapped_variant, []) - assert calibration is None - assert functional_range is None - - def test_returns_single_calibration(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that single calibration is returned.""" - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - assert calibration is not None - assert calibration == calibrations[0] - assert functional_range is not None - - def test_selects_strongest_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that calibration with strongest evidence is selected.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return different evidence strengths - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ( - MagicMock(label="Strong"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.VERY_STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with VERY_STRONG evidence - assert calibration == calibration2 - assert functional_range.label == "Strong" - - def test_defaults_to_uncertain_on_tie(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that uncertain significance is returned when benign and pathogenic evidence tie.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength but different criteria (pathogenic vs benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Pathogenic"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration but None for range to indicate uncertain significance - assert calibration == calibration1 - assert functional_range is None - - def test_returns_classification_when_all_tied_are_same_type( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that classification is returned normally when all tied candidates are the same type.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock to return same evidence strength and same type of criteria (both benign) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - ( - MagicMock(label="Benign1"), - VariantPathogenicityEvidenceLine.Criterion.BS3, - StrengthOfEvidenceProvided.STRONG, - ), - ( - MagicMock(label="Benign2"), - VariantPathogenicityEvidenceLine.Criterion.BP1, - StrengthOfEvidenceProvided.STRONG, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with its range (no conflict, so normal classification) - assert calibration == calibration1 - assert functional_range.label == "Benign1" - - def test_handles_none_evidence_strength(self, mock_mapped_variant_with_pathogenicity_calibration_score_set): - """Test that None evidence strength is handled correctly.""" - from copy import deepcopy - from unittest.mock import MagicMock, patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibration1 = mapped_variant.variant.score_set.score_calibrations[0] - calibration2 = deepcopy(calibration1) - calibration2.id = 999 - calibrations = [calibration1, calibration2] - - # Mock with None and actual strength - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - side_effect=[ - (MagicMock(label="No Strength"), VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ( - MagicMock(label="Moderate"), - VariantPathogenicityEvidenceLine.Criterion.PS3, - StrengthOfEvidenceProvided.MODERATE, - ), - ], - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return the one with actual strength (second) - assert calibration == calibration2 - assert functional_range.label == "Moderate" - - def test_returns_first_calibration_when_no_variants_in_ranges( - self, mock_mapped_variant_with_pathogenicity_calibration_score_set - ): - """Test that first calibration with None range is returned when variant is not in any functional range.""" - from unittest.mock import patch - - from ga4gh.va_spec.acmg_2015 import VariantPathogenicityEvidenceLine - - mapped_variant = mock_mapped_variant_with_pathogenicity_calibration_score_set - calibrations = mapped_variant.variant.score_set.score_calibrations - - # Mock to return None range for all calibrations (variant not in any range) - with patch( - "mavedb.lib.annotation.util.pathogenicity_classification_of_variant", - return_value=(None, VariantPathogenicityEvidenceLine.Criterion.PS3, None), - ): - calibration, functional_range = select_strongest_pathogenicity_calibration(mapped_variant, calibrations) - - # Should return first calibration with None range (indicating variant not in any range) - assert calibration == calibrations[0] - assert functional_range is None diff --git a/tests/lib/test_vrs.py b/tests/lib/test_vrs.py new file mode 100644 index 000000000..fae8dda67 --- /dev/null +++ b/tests/lib/test_vrs.py @@ -0,0 +1,96 @@ +# ruff: noqa: E402 + +"""Tests for mavedb.lib.vrs — deserialization of stored post_mapped dicts into GA4GH VRS objects.""" + +from copy import deepcopy + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from tests.helpers.constants import ( + TEST_SEQUENCE_LOCATION_ACCESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, +) + + +@pytest.mark.unit +class TestVariationExtractionUnit: + @pytest.mark.parametrize( + "variation_version", + [{"variation": TEST_VALID_POST_MAPPED_VRS_ALLELE}, TEST_VALID_POST_MAPPED_VRS_ALLELE], + ids=["vrs13_wrapped_variation", "vrs2_direct_allele"], + ) + def test_vrs_object_from_post_mapped_variation(self, variation_version): + result = vrs_object_from_mapped_variant(variation_version).model_dump() + + assert result["location"]["id"] == TEST_SEQUENCE_LOCATION_ACCESSION + assert result["location"]["start"] == 5 + assert result["location"]["end"] == 6 + + @pytest.mark.parametrize( + "allele_dict, expected_state_type, expected_state_fields", + [ + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + "LiteralSequenceExpression", + {"sequence": "F"}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + "ReferenceLengthExpression", + {"length": 5, "repeatSubunitLength": 5}, + ), + ( + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + "LengthExpression", + {"length": 5}, + ), + ], + ids=["lse_state", "rle_state", "length_expression_state"], + ) + def test_allele_state_type_is_deserialized_correctly(self, allele_dict, expected_state_type, expected_state_fields): + result = vrs_object_from_mapped_variant(allele_dict).model_dump() + + assert result["state"]["type"] == expected_state_type + for k, v in expected_state_fields.items(): + assert result["state"][k] == v + + def test_unknown_allele_state_type_raises_value_error(self): + allele_dict = { + **TEST_VALID_POST_MAPPED_VRS_ALLELE, + "state": {"type": "UnknownFutureExpression", "someField": "someValue"}, + } + + with pytest.raises(ValueError, match="Unsupported VRS Allele state type"): + vrs_object_from_mapped_variant(allele_dict) + + @pytest.mark.parametrize( + "member_dict, expected_state_type", + [ + (TEST_VALID_POST_MAPPED_VRS_ALLELE, "LiteralSequenceExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, "ReferenceLengthExpression"), + (TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, "LengthExpression"), + ], + ids=["lse_members", "rle_members", "length_expression_members"], + ) + def test_cis_phased_block_member_state_types_are_deserialized_correctly(self, member_dict, expected_state_type): + mapping_results = {"type": "CisPhasedBlock", "members": [deepcopy(member_dict), deepcopy(member_dict)]} + + result = vrs_object_from_mapped_variant(mapping_results).model_dump() + + assert result["type"] == "CisPhasedBlock" + assert len(result["members"]) == 2 + assert result["members"][0]["state"]["type"] == expected_state_type + + +@pytest.mark.integration +class TestVrsIntegration: + def test_vrs_object_from_persisted_post_mapped(self, setup_lib_db_with_mapped_variant): + variation = vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + + assert variation is not None + assert variation.model_dump().get("type") in {"Allele", "CisPhasedBlock"} From a8363a6f0a3d52de81ec74655496ef17aa60e995 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 10:25:24 -0700 Subject: [PATCH 78/93] feat(annotation): serve variant annotation from the Allele substrate Move annotation serving off the legacy MappedVariant model onto the MappingRecord/Allele substrate written by the mapping pipeline. New score sets, whose mapping data lands only on the new substrate, previously enumerated no annotatable variants and silently returned nothing. - Replace get_current_mapped_variants_for_annotation with get_annotatable_variants, querying live MappingRecord/Allele links and threading an as_of instant for temporal reconstruction. - Thread as_of end to end through the three streaming annotation endpoints: into get_annotatable_variants (set enumeration) as well as the per-variant context, so the set and its annotations share one instant and cannot drift. - Return 200 with an empty stream when a score set exists but has no annotatable variants (never mapped, or none live at as_of); 404 is reserved for an unresolvable URN or a permission failure, so as_of acts as a filter and never manufactures a 404. - Derive mavedb_vrs_contribution provenance from the live MappingRecord via VariantAnnotationContext instead of MappedVariant. - Add find_variants_by_vrs_identifier over Allele.vrs_digest, and repoint the public-data export onto get_annotatable_variants. --- src/mavedb/lib/alleles.py | 29 ++++ src/mavedb/lib/annotation/contribution.py | 12 +- src/mavedb/lib/score_sets.py | 79 ++++++---- src/mavedb/routers/score_sets.py | 180 +++++++++++++--------- src/mavedb/scripts/export_public_data.py | 12 +- tests/helpers/util/score_set.py | 34 ++++ tests/lib/annotation/test_contribution.py | 26 ++-- tests/routers/test_score_set.py | 110 +++++++++---- 8 files changed, 336 insertions(+), 146 deletions(-) diff --git a/src/mavedb/lib/alleles.py b/src/mavedb/lib/alleles.py index bf6a340d4..08f13ef0b 100644 --- a/src/mavedb/lib/alleles.py +++ b/src/mavedb/lib/alleles.py @@ -17,6 +17,7 @@ from mavedb.models.allele import Allele from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant def get_live_record_allele_links( @@ -51,6 +52,34 @@ def get_live_record_allele_links( ) +def find_variants_by_vrs_identifier( + db: Session, identifier: str, *, as_of: Optional[datetime] = None +) -> list[tuple[Variant, Allele]]: + """Resolve a GA4GH VRS identifier to the variants whose mapping links an allele bearing it. + + Matches ``identifier`` against a deduplicated allele's ``vrs_digest``, then walks ``Allele → + MappingRecordAllele → MappingRecord → Variant``. Because alleles are deduplicated by ``vrs_digest`` + and shared across variants/score sets, one identifier can resolve to several variants; the + paired allele carries the ClinGen id + level the caller surfaces alongside the URN. + + The live link + record set (``valid_to IS NULL`` / ``as_of``) is used to filter results. Returns distinct + ``(variant, matched_allele)`` pairs. Include an ``as_of`` timestamp to reconstruct the set as it stood + at a past instant. + """ + stmt = ( + select(Variant, Allele) + .select_from(Allele) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Allele.vrs_digest == identifier) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + ) + + return [(row[0], row[1]) for row in db.execute(stmt).unique().all()] + + def get_allele_translations(db: Session, allele_id: int, *, as_of: Optional[datetime] = None) -> list[Allele]: """Return the cross-layer equivalence set of ``allele_id``: every allele co-linked to a ``MappingRecord`` that links it (the anchor allele itself included), spanning the genomic, coding, diff --git a/src/mavedb/lib/annotation/contribution.py b/src/mavedb/lib/annotation/contribution.py index 4b0de881f..839721b5f 100644 --- a/src/mavedb/lib/annotation/contribution.py +++ b/src/mavedb/lib/annotation/contribution.py @@ -9,8 +9,8 @@ mavedb_user_agent, mavedb_vrs_agent, ) +from mavedb.lib.annotation.context import VariantAnnotationContext from mavedb.lib.types.annotation import ResourceWithCreationModificationDates -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User @@ -31,17 +31,17 @@ def mavedb_api_contribution() -> Contribution: ) -def mavedb_vrs_contribution(mapped_variant: MappedVariant) -> Contribution: +def mavedb_vrs_contribution(context: VariantAnnotationContext) -> Contribution: """ Create a [VA Contribution](https://va-ga4gh.readthedocs.io/en/latest/core-information-model/entities/activities/contribution.html#contribution) - object from the provided mapped variant. + object from the variant's live mapping record (the VRS mapper's provenance). """ return Contribution( name="MaveDB VRS Mapper", description="Contribution from the MaveDB VRS mapping software", - # Guaranteed to be a str via DB constraints. - contributor=mavedb_vrs_agent(mapped_variant.mapping_api_version), # type: ignore - date=mapped_variant.mapped_date, # type: ignore + # Both guaranteed non-null via DB constraints on MappingRecord. + contributor=mavedb_vrs_agent(context.record.mapping_api_version), # type: ignore + date=context.record.mapped_date, # type: ignore activityType="human genome sequence mapping process", ) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 76cbab9e6..3c1559510 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -3,6 +3,7 @@ import logging import re from collections import Counter, defaultdict +from datetime import datetime from operator import attrgetter from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence @@ -10,8 +11,8 @@ import pandas as pd from pandas.testing import assert_index_equal from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -29,6 +30,9 @@ from mavedb.lib.validation.constants.general import null_values_list from mavedb.lib.validation.utilities import is_null as validate_is_null from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.allele import Allele +from mavedb.models.clinical_control import ClinvarControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,10 +42,10 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier from mavedb.models.refseq_offset import RefseqOffset @@ -57,7 +61,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -554,35 +558,50 @@ def find_publish_or_private_superseded_score_set_tail( return score_set -def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) -> Sequence[MappedVariant]: - """ - Load the current mapped variants for a score set with the relationships required to build VA-Spec - annotations eagerly loaded. - - This is the single source of truth for the eager-load shape shared by the annotated-variant - streaming endpoints and the public data export. The annotation builders reach through - ``MappedVariant.variant.score_set`` for publications, contributors, license, experiment, and score - calibrations, so each of those is loaded up front to avoid per-variant lazy loads. +def get_annotatable_variants( + db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None +) -> Sequence[Variant]: + """Load a score set's annotatable variants from its variant's allele-graph, with the score-set + relationships the VA-Spec builders need eagerly loaded. + + A variant is *annotatable* when it has a live ``MappingRecord`` whose authoritative allele carries a + ``post_mapped`` VRS — exactly the condition under which :func:`annotation.context.variant_annotation_context` + yields a context. This is the single source of truth for the eager-load shape shared by the + annotated-variant streaming endpoints and the public data export. The annotation builders reach through + ``variant.score_set`` for publications, contributors, license, experiment, and score calibrations, so + each of those is loaded up front to avoid per-variant lazy loads. + + ``as_of`` reconstructs the annotatable set as it stood at a past instant: the same half-open + ``[valid_from, valid_to)`` predicate is applied to both the record and the authoritative link, so the + set is evaluated at one moment (matching :func:`annotation.context.variant_annotation_context`'s own + ``as_of``, so a per-variant context can be rebuilt at the same instant downstream). Defaults to the + currently-live rows. """ return ( - db.query(MappedVariant) - .join(MappedVariant.variant) - .join(Variant.score_set) - .filter(Variant.score_set_id == score_set.id) - .filter(MappedVariant.current.is_(True)) - .options( - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.publication_identifier_associations), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.created_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.modified_by), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.license), - contains_eager(MappedVariant.variant).contains_eager(Variant.score_set).selectinload(ScoreSet.experiment), - contains_eager(MappedVariant.variant) - .contains_eager(Variant.score_set) - .selectinload(ScoreSet.score_calibrations), + db.scalars( + select(Variant) + .join(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .join( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .join(Allele, and_(Allele.id == MappingRecordAllele.allele_id, Allele.post_mapped.isnot(None))) + .where(Variant.score_set_id == score_set.id) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.publication_identifier_associations), + selectinload(Variant.score_set).selectinload(ScoreSet.created_by), + selectinload(Variant.score_set).selectinload(ScoreSet.modified_by), + selectinload(Variant.score_set).selectinload(ScoreSet.license), + selectinload(Variant.score_set).selectinload(ScoreSet.experiment), + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + ) + .distinct() ) + .unique() .all() ) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 7a68e8550..506ed0b5b 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -27,12 +27,14 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.context import variant_annotation_context from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authorization import ( get_current_user, require_current_user, require_current_user_with_email, ) +from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets @@ -47,7 +49,6 @@ logging_context, save_to_logging_context, ) -from mavedb.lib.clinical_controls import get_clinical_control_options, get_clinical_controls_with_variant_urns from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -56,7 +57,7 @@ csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, - get_current_mapped_variants_for_annotation, + get_annotatable_variants, get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, @@ -1185,10 +1186,13 @@ def get_score_set_mapped_variants( return mapped_variants -def _stream_generated_annotations(mapped_variants, annotation_function): +def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. + Builds each variant's :class:`VariantAnnotationContext` from the mapping-record substrate and passes it + to ``annotation_function``; variants with no live mapping data yield a null annotation. + Metadata should be provided via HTTP headers: - X-Total-Count: Total number of variants - X-Processing-Started: ISO timestamp when processing began @@ -1198,20 +1202,21 @@ def _stream_generated_annotations(mapped_variants, annotation_function): consumed via Server-Sent Events if needed. """ start_time = time.time() - total_variants = len(mapped_variants) + total_variants = len(variants) processed_count = 0 - logger.info(f"Starting streaming processing of {total_variants} mapped variants") + logger.info(f"Starting streaming processing of {total_variants} variants") - for i, mv in enumerate(mapped_variants): + for i, variant in enumerate(variants): try: - annotation = annotation_function(mv) + context = variant_annotation_context(db, variant, as_of=as_of) + annotation = annotation_function(context) if context is not None else None except MappingDataDoesntExistException: - logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") + logger.debug(f"Mapping data does not exist for variant {variant.urn}.") annotation = None # Send pure result data (no wrapper) result = { - "variant_urn": mv.variant.urn, + "variant_urn": variant.urn, "annotation": annotation.model_dump(exclude_none=True) if annotation else None, } yield json.dumps(result, default=str) + "\n" @@ -1258,11 +1263,11 @@ def _stream_generated_annotations(mapped_variants, annotation_function): status_code=200, response_model=dict[str, Optional[VariantPathogenicityStatement]], response_model_exclude_none=True, - summary="Get pathogenicity statement annotations for mapped variants within a score set", + summary="Get pathogenicity statement annotations for variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream pathogenicity statement annotations for mapped variants.", + "description": "Stream pathogenicity statement annotations for variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1270,22 +1275,31 @@ def _stream_generated_annotations(mapped_variants, annotation_function): def get_score_set_annotated_variants( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: """ Retrieve annotated variants with pathogenicity statements for a given score set. - This endpoint streams pathogenicity evidence lines for all current mapped variants + This endpoint streams pathogenicity evidence lines for all current annotated variants associated with a specific score set. The response is returned as newline-delimited JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to an annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Pathogenicity evidence line details } @@ -1301,23 +1315,27 @@ def get_score_set_annotated_variants( Returns: Any: StreamingResponse containing newline-delimited JSON with pathogenicity - evidence lines for each mapped variant. Response includes headers with + evidence lines for each annotated variant. Response includes headers with total count, processing start time, and stream type information. + A score set that exists but has no annotatable variants (never mapped, or none live at ``as_of``) + streams an empty body with ``X-Total-Count: 0`` — an empty collection, not a 404. + Raises: HTTPException: 404 error if the score set with the given URN is not found. - HTTPException: 404 error if no mapped variants are associated with the score set. HTTPException: 403 error if the user lacks READ permissions for the score set. Note: This function logs the request context and validates user permissions before - processing. Only current (non-historical) mapped variants are included in - the response. + processing. Use the `as_of` parameter to reconstruct the molecular layer as it stood at a specific + instant, over the variant's fixed score. The response is streamed to allow for efficient handling + of large datasets, and progress updates are logged for monitoring purposes. """ save_to_logging_context( { "requested_resource": urn, "resource_property": "annotated-variants/pathogenicity-statement", + "as_of": as_of, } ) @@ -1331,20 +1349,17 @@ def get_score_set_annotated_variants( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct evidence lines.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_pathogenicity_statement), + _stream_generated_annotations(db, variants, variant_pathogenicity_statement, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "pathogenicity-evidence-line", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1357,11 +1372,11 @@ def get_score_set_annotated_variants( status_code=200, response_model=dict[str, Optional[Statement]], response_model_exclude_none=True, - summary="Get functional impact statement annotations for mapped variants within a score set", + summary="Get functional impact statement annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional impact statement annotations for mapped variants.", + "description": "Stream functional impact statement annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1369,22 +1384,31 @@ def get_score_set_annotated_variants( def get_score_set_annotated_variants_functional_statement( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional impact statements for annotated variants in a score set. - This endpoint streams functional impact statements for all current mapped variants + This endpoint streams functional impact statements for all current annotated variants associated with a specific score set. The response is delivered as newline-delimited JSON (NDJSON) format. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to an annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional impact statement details } @@ -1398,20 +1422,28 @@ def get_score_set_annotated_variants_functional_statement( Returns: StreamingResponse: NDJSON stream containing functional impact statements for each - mapped variant. Response includes headers with total count, processing start time, + annotated variant. Response includes headers with total count, processing start time, and stream type information. Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Note: - Only current (non-historical) mapped variants are included in the response. - The function requires appropriate read permissions on the score set. + The function requires appropriate read permissions on the score set. Use the `as_of` + parameter to reconstruct the molecular layer as it stood at a specific instant, over + the variant's fixed score. The response is streamed to allow for efficient handling of + large datasets, and progress updates are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/functional-statement"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/functional-statement", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1423,20 +1455,17 @@ def get_score_set_annotated_variants_functional_statement( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct functional impact statements.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_functional_impact_statement), + _stream_generated_annotations(db, variants, variant_functional_impact_statement, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-impact-statement", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", @@ -1449,11 +1478,11 @@ def get_score_set_annotated_variants_functional_statement( status_code=200, response_model=dict[str, Optional[ExperimentalVariantFunctionalImpactStudyResult]], response_model_exclude_none=True, - summary="Get functional study result annotations for mapped variants within a score set", + summary="Get functional study result annotations for annotated variants within a score set", responses={ 200: { "content": {"application/x-ndjson": {}}, - "description": "Stream functional study result annotations for mapped variants.", + "description": "Stream functional study result annotations for annotated variants.", }, **ACCESS_CONTROL_ERROR_RESPONSES, }, @@ -1461,22 +1490,31 @@ def get_score_set_annotated_variants_functional_statement( def get_score_set_annotated_variants_functional_study_result( *, urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ): """ Retrieve functional study results for annotated variants in a score set. - This endpoint streams functional study result annotations for all current mapped variants + This endpoint streams functional study result annotations for all current annotated variants associated with a specific score set. The results are returned as newline-delimited JSON (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON + Each line in the response corresponds to a annotated variant and contains a JSON object with the following structure: ``` { - "variant_urn": "", + "variant_urn": "", "annotation": { ... // Functional study result details } @@ -1491,7 +1529,7 @@ def get_score_set_annotated_variants_functional_study_result( Returns: StreamingResponse: A streaming response containing functional study results in NDJSON format. Headers include: - - X-Total-Count: Total number of mapped variants being streamed + - X-Total-Count: Total number of annotated variants being streamed - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Set to "functional-study-result" - Access-Control-Expose-Headers: Exposed headers for CORS @@ -1499,15 +1537,22 @@ def get_score_set_annotated_variants_functional_study_result( Raises: HTTPException: - 404 if the score set with the given URN is not found - - 404 if no mapped variants are associated with the score set + - 404 if no annotated variants are associated with the score set - 403 if the user lacks READ permission for the score set Notes: - - Only returns current mapped variants (MappedVariant.current == True) - - Eagerly loads related ScoreSet data including publications, users, license, and experiment - - Logs requests and errors for monitoring and debugging purposes + - The `as_of` parameter allows reconstruction of the molecular layer as it stood at a specific + instant, over the variant's fixed score. It is ISO 8601 formatted and ideally timezone-aware. + - The response is streamed to allow for efficient handling of large datasets, and progress updates + are logged for monitoring purposes. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "annotated-variants/study-result"}) + save_to_logging_context( + { + "requested_resource": urn, + "resource_property": "annotated-variants/study-result", + "as_of": as_of, + } + ) score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: @@ -1519,20 +1564,17 @@ def get_score_set_annotated_variants_functional_study_result( assert_permission(user_data, score_set, Action.READ) - mapped_variants = get_current_mapped_variants_for_annotation(db, score_set) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variants associated with score set URN {urn} were found. Could not construct study results.", - ) + variants = get_annotatable_variants(db, score_set, as_of=as_of) + # An existing score set with no annotatable variants (never mapped, or none live at as_of) streams an + # empty NDJSON body with X-Total-Count: 0 — an empty collection, not a 404. 404 is reserved for an + # unresolvable URN or a permission failure, never for a filter (as_of) matching nothing. return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_study_result), + _stream_generated_annotations(db, variants, variant_study_result, as_of=as_of), media_type="application/x-ndjson", headers={ - "X-Total-Count": str(len(mapped_variants)), + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), "X-Processing-Started": datetime.now().isoformat(), "X-Stream-Type": "functional-study-result", "Access-Control-Expose-Headers": "X-Total-Count, X-Processing-Started, X-Stream-Type", diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index cedc66608..07098cf4f 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -30,7 +30,8 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.score_sets import get_annotatable_variants, get_score_set_variants_as_csv from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -211,16 +212,17 @@ def export_public_data(db: Session): # Write VA-Spec annotations NDJSON — mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* # streams, emitting one record per current mapped variant at its highest materialized VA level. - annotated_variants = get_current_mapped_variants_for_annotation(db, score_set) + annotatable_variants = get_annotatable_variants(db, score_set) va_lines = [] num_annotations = 0 - for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv) + for variant in annotatable_variants: + context = variant_annotation_context(db, variant) + annotation = variant_highest_level_annotation(context) if context is not None else None if annotation is not None: num_annotations += 1 record = { - "variant_urn": mv.variant.urn, + "variant_urn": variant.urn, "annotation": annotation.model_dump(exclude_none=True) if annotation else None, } va_lines.append(json.dumps(record, default=str)) diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 067c627fe..7e9022db7 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -205,6 +205,40 @@ def create_acc_score_set_with_variants( return score_set +def seed_annotation_substrate(db, score_set, *, skip_first=False): + """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. + + The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` + substrate (not the legacy ``MappedVariant``), so a variant needs a live record with an + authoritative ``post_mapped`` allele before a study result / statement can be constructed. This + mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is + deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) + + With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + "some variants were not mapped" cases). Returns the score set's variants in id order. + """ + variants = db.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + for variant in variants[1:] if skip_first else variants: + seed_mapping_record( + db, + variant, + alleles=[ + AlleleSpec( + digest=f"va-allele-{variant.id}", + is_authoritative=True, + clingen_allele_id=f"CA{variant.id}", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + return variants + + def link_clinical_controls_to_alleles(db, score_set): """Seed the new-model annotation graph for a score set's first two variants and link the two seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. diff --git a/tests/lib/annotation/test_contribution.py b/tests/lib/annotation/test_contribution.py index 51597db37..60990baf6 100644 --- a/tests/lib/annotation/test_contribution.py +++ b/tests/lib/annotation/test_contribution.py @@ -23,14 +23,17 @@ mavedb_score_calibration_contribution, mavedb_vrs_contribution, ) +from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.user import User +from tests.helpers.constants import TEST_VALID_POST_MAPPED_VRS_ALLELE from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_resource_with_dates, create_mock_score_calibration, create_mock_user, ) +from tests.lib.annotation.conftest import annotation_context_for @pytest.mark.unit @@ -85,14 +88,14 @@ class TestMavedbVrsContributionUnit: def test_returns_contribution_object(self): """Test that function returns proper Contribution object.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) def test_has_correct_name_and_description(self): """Test that contribution has correct name and description.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.name == "MaveDB VRS Mapper" assert contribution.description == "Contribution from the MaveDB VRS mapping software" @@ -100,7 +103,7 @@ def test_has_correct_name_and_description(self): def test_has_correct_activity_type(self): """Test that contribution has correct activity type.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.activityType == "human genome sequence mapping process" @@ -108,14 +111,14 @@ def test_uses_mapped_variant_date(self): """Test that contribution uses mapped variant date.""" test_date = datetime(2024, 3, 15, 12, 0, 0) mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.date == test_date def test_has_contributor(self): """Test that contribution has a contributor.""" mapped_variant = create_mock_mapped_variant() - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert contribution.contributor is not None @@ -123,7 +126,7 @@ def test_has_contributor(self): def test_various_api_versions(self, api_version): """Test function works with various API versions.""" mapped_variant = create_mock_mapped_variant(mapping_api_version=api_version) - contribution = mavedb_vrs_contribution(mapped_variant) + contribution = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert isinstance(contribution, Contribution) assert contribution.name == "MaveDB VRS Mapper" @@ -316,7 +319,12 @@ class TestContributionIntegration: def test_contributions_with_real_db_objects(self, session, setup_lib_db_with_mapped_variant): """Test contribution creation from persisted SQLAlchemy objects.""" mapped_variant = setup_lib_db_with_mapped_variant - vrs_contribution = mavedb_vrs_contribution(mapped_variant) + # The VRS contribution derives from the record's mapper provenance, not the subject; hand the + # context an explicit subject so it doesn't depend on this fixture's minimal post_mapped. + context = annotation_context_for( + mapped_variant, subject_variant=vrs_object_from_mapped_variant(TEST_VALID_POST_MAPPED_VRS_ALLELE) + ) + vrs_contribution = mavedb_vrs_contribution(context) creator = session.query(User).first() score_set = mapped_variant.variant.score_set @@ -350,7 +358,7 @@ def test_all_contributions_return_consistent_structure(self): api_contrib = mavedb_api_contribution() mapped_variant = create_mock_mapped_variant() - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) calibration = create_mock_score_calibration() cal_contrib = mavedb_score_calibration_contribution(calibration) @@ -376,7 +384,7 @@ def test_date_formatting_consistency(self): # VRS contribution mapped_variant = create_mock_mapped_variant(mapped_date=test_date) - vrs_contrib = mavedb_vrs_contribution(mapped_variant) + vrs_contrib = mavedb_vrs_contribution(annotation_context_for(mapped_variant)) assert vrs_contrib.date == test_date # Score calibration contribution diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 942f032c7..11a07bf51 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -85,10 +85,10 @@ link_clinvar_control_to_mapped_variant, link_gnomad_variants_to_mapped_variants, publish_score_set, + seed_annotation_substrate, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - clear_first_mapped_variant_post_mapped, create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -4055,9 +4055,14 @@ def test_cannot_get_annotated_variants_for_nonexistent_score_set(client, setup_r @pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) -def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( +def test_get_annotated_variants_for_score_set_with_no_mapped_variants_streams_empty( client, session, data_provider, data_files, setup_router_db, annotation_type ): + """A score set that exists but has no annotatable variants is an empty collection, not a 404. + + The route resolves (the URN names a real, readable score set), so it returns 200 with an empty NDJSON + body and ``X-Total-Count: 0``. 404 is reserved for an unresolvable URN or a permission failure. + """ experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -4082,13 +4087,47 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( assert "hgvs_splice" not in columns response = client.get(f"/api/v1/score-sets/{publish_score_set['urn']}/annotated-variants/{annotation_type}") - response_data = response.json() - assert response.status_code == 404 - assert ( - f"No mapped variants associated with score set URN {publish_score_set['urn']} were found" - in response_data["detail"] + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +@pytest.mark.parametrize("annotation_type", ["pathogenicity-statement", "functional-statement", "study-result"]) +def test_annotated_variants_honor_as_of(client, session, data_provider, data_files, setup_router_db, annotation_type): + """The streaming annotation routes time-travel their annotatable set via ``as_of``. + + ``as_of`` is threaded into ``get_annotatable_variants`` (which allele links are live at that instant), + not only into the per-variant context. So a far-future instant sees the freshly-seeded substrate as + live (the full set streams, and the route echoes the instant on ``X-As-Of``); a far-past instant sees + no live mapping data (the annotatable set is empty). Either way the route returns 200 — ``as_of`` is a + filter, so an empty match is an empty collection, never a 404. This pins the wiring so the set + enumeration and the per-variant annotation cannot drift to different instants. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/{annotation_type}" + + # Far future: the seeded mapping records are live, so the whole set streams and X-As-Of echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + # Far past: nothing was live yet, so the annotatable set is empty — an empty 200 stream, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + + assert past_response.status_code == 200 + assert datetime.fromisoformat(past_response.headers["X-As-Of"]) == past + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] # Tests that annotated variants of the correct type are returned when appropriate. The contents of these @@ -4119,6 +4158,7 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) # The contents of the annotated variants objects should be tested in more detail elsewhere. response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") @@ -4150,6 +4190,10 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_thre data_files / "scores.csv", ) + # Variants are mapped on the allele substrate, so they are annotatable; the null annotation comes from + # the missing calibration/thresholds, not from missing mapping. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4173,6 +4217,9 @@ def test_nonetype_annotated_pathogenicity_evidence_lines_for_score_set_when_cali data_files / "scores.csv", ) + # Variants are mapped on the allele substrate (so annotatable); the null comes from the absent calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) @@ -4209,21 +4256,21 @@ def test_get_annotated_pathogenicity_evidence_lines_for_score_set_when_some_vari client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/pathogenicity-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") + assert f"Variant pathogenicity statement for {variant_urn}" in annotated_variant.get("description", "") @pytest.mark.parametrize( @@ -4250,6 +4297,7 @@ def test_get_annotated_functional_impact_statement_for_score_set( create_publish_and_promote_score_calibration( client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4283,6 +4331,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_calib }, ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent promoted calibration. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4306,6 +4357,9 @@ def test_nonetype_annotated_functional_impact_statement_for_score_set_when_thres data_files / "scores.csv", ) + # Mapped on the allele substrate (so annotatable); the null comes from the absent calibration/ranges. + seed_annotation_substrate(session, score_set) + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) @@ -4342,21 +4396,20 @@ def test_get_annotated_functional_impact_statement_for_score_set_when_some_varia client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/functional-statement") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "Statement" + assert annotated_variant.get("type") == "Statement" @pytest.mark.parametrize( @@ -4375,6 +4428,7 @@ def test_get_annotated_functional_study_result_for_score_set( experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4407,6 +4461,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4439,6 +4494,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_ranges_not_ "scoreRanges": camelize([TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_PATHOGENICITY_SCORE_CALIBRATION]), }, ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4462,6 +4518,7 @@ def test_annotated_functional_study_result_exists_for_score_set_when_thresholds_ experiment["urn"], data_files / "scores.csv", ) + seed_annotation_substrate(session, score_set) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) @@ -4495,21 +4552,20 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian }, ) - first_var = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + first_var_urn = seed_annotation_substrate(session, score_set, skip_first=True)[0].urn response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") response_data = parse_ndjson_response(response) assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] + # The first variant is unmapped on the allele substrate, so it is not annotatable and is omitted from + # the stream entirely (rather than emitted with a null annotation). + assert len(response_data) == score_set["numVariants"] - 1 + assert first_var_urn not in {annotation_response.get("variant_urn") for annotation_response in response_data} for annotation_response in response_data: - variant_urn = annotation_response.get("variant_urn") annotated_variant = annotation_response.get("annotation") - if variant_urn == first_var.urn: - assert annotated_variant is None - else: - assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" ######################################################################################################################## From 599c8e018a576356f25347684f47ae90d9e233d5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 10:42:03 -0700 Subject: [PATCH 79/93] feat(variants): relocate VA-Spec and VRS-lookup endpoints; drop /mapped-variants router Move the single-variant annotation surface onto the variants router, served from the MappingRecord/Allele substrate, and remove the legacy mapped_variant router entirely. - Add GET /variants/{urn}/va/{study-result,functional-statement, pathogenicity-statement}, each building a VA-Spec resource from the variant's live mapping context and threading as_of for temporal reconstruction. - Add GET /variants/vrs/{identifier}: resolve a GA4GH VRS id to the readable variants whose mapping links that allele, filtered by as_of. - Delete routers/mapped_variant.py and its registration in server_main. 404 semantics follow the collection-vs-resource split: the VRS lookup is collection-shaped, so no readable match returns 200 with an empty list (an absent id and a private-only match are indistinguishable, hiding existence); the single derived /va/* resources 404 when the variant is unknown or the statement does not exist at the requested instant. --- src/mavedb/routers/mapped_variant.py | 262 ---------- src/mavedb/routers/variants.py | 269 +++++++++- src/mavedb/server_main.py | 3 - src/mavedb/view_models/variant.py | 14 + tests/routers/test_mapped_variants.py | 620 ----------------------- tests/routers/test_variant.py | 20 +- tests/routers/test_variant_annotation.py | 352 +++++++++++++ 7 files changed, 639 insertions(+), 901 deletions(-) delete mode 100644 src/mavedb/routers/mapped_variant.py delete mode 100644 tests/routers/test_mapped_variants.py create mode 100644 tests/routers/test_variant_annotation.py diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py deleted file mode 100644 index 7b97b304c..000000000 --- a/src/mavedb/routers/mapped_variant.py +++ /dev/null @@ -1,262 +0,0 @@ -import logging -from typing import Annotated, Any, Optional - -from fastapi import APIRouter, Depends, Path -from fastapi.exceptions import HTTPException -from ga4gh.core.identifiers import GA4GH_IR_REGEXP -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound -from sqlalchemy.orm import Session - -from mavedb import deps -from mavedb.lib.annotation.annotate import ( - variant_functional_impact_statement, - variant_pathogenicity_statement, - variant_study_result, -) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user -from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - logging_context, - save_to_logging_context, -) -from mavedb.lib.permissions import Action, assert_permission, has_permission -from mavedb.lib.types.authentication import UserData -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.variant import Variant -from mavedb.routers.shared import ACCESS_CONTROL_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX -from mavedb.view_models import mapped_variant - -TAG_NAME = "Mapped Variants" - -logger = logging.getLogger(__name__) - - -async def fetch_mapped_variant_by_variant_urn(db: Session, user: Optional[UserData], urn: str) -> MappedVariant: - """ - We may combine this function back to show_mapped_variant if none of any new function call it. - Fetch one mapped variant by variant URN. - - :param db: An active database session. - :param urn: The variant URN. - :return: The mapped variant. - - :raises HTTPException: If the mapped variant is not found or if multiple variants are found. - """ - try: - item = ( - db.execute(select(MappedVariant).join(Variant).where(Variant.urn == urn, MappedVariant.current.is_(True))) - .scalars() - .one_or_none() - ) - except MultipleResultsFound: - logger.info( - msg="Could not fetch the requested mapped variant; Multiple such variants exist.", extra=logging_context() - ) - raise HTTPException(status_code=500, detail=f"Multiple variants with URN {urn} were found.") - - if not item: - logger.info( - msg="Could not fetch the requested mapped variant; No such mapped variants exist.", extra=logging_context() - ) - - raise HTTPException(status_code=404, detail=f"Mapped variant with URN {urn} not found") - - # Base mapped variant read permission on the score set in which it is contained. - assert_permission(user, item.variant.score_set, Action.READ) - return item - - -router = APIRouter( - prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", - tags=[TAG_NAME], - responses={**PUBLIC_ERROR_RESPONSES}, - route_class=LoggedRoute, -) - -metadata = { - "name": TAG_NAME, - "description": "Retrieve mapped variants and their associated variant annotations.", -} - - -@router.get( - "/{urn}", - status_code=200, - response_model=mapped_variant.MappedVariantWithMappingDetails, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variant by URN", -) -async def show_mapped_variant( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Any: - """ - Fetch a single mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - return await fetch_mapped_variant_by_variant_urn(db, user, urn) - - -@router.get( - "/{urn}/va/study-result", - status_code=200, - response_model=ExperimentalVariantFunctionalImpactStudyResult, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec StudyResult from a mapped variant", -) -async def show_mapped_variant_study_result( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> ExperimentalVariantFunctionalImpactStudyResult: - """ - Construct a single VA-Spec StudyResult from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - return variant_study_result(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg=f"Could not construct a study result for mapped variant {urn}: {e}", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No study result exists for mapped variant {urn}: {e}") - - -# TODO#416: For now, this route supports only one statement per mapped variant. Eventually, we should support the possibility of multiple statements. -@router.get( - "/{urn}/va/functional-statement", - status_code=200, - response_model=Statement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec Statement from a mapped variant", -) -async def show_mapped_variant_functional_impact_statement( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> Statement: - """ - Construct a single VA-Spec Statement from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - functional_impact = variant_functional_impact_statement(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, detail=f"No functional impact statement exists for mapped variant {urn}: {e}" - ) - - if not functional_impact: - logger.info( - msg="Could not construct a functional impact statement for this mapped variant. Variant does not have sufficient evidence to evaluate its functional impact.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No functional impact statement exists for mapped variant {urn}. Variant does not have sufficient evidence to evaluate its functional impact.", - ) - - return functional_impact - - -# TODO#416: For now, this route supports only one evidence line per mapped variant. Eventually, we should support the possibility of multiple evidence lines. -@router.get( - "/{urn}/va/pathogenicity-statement", - status_code=200, - response_model=VariantPathogenicityStatement, - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Construct a VA-Spec EvidenceLine from a mapped variant", -) -async def show_mapped_variant_acmg_evidence_line( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) -) -> VariantPathogenicityStatement: - """ - Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. - """ - save_to_logging_context({"requested_resource": urn}) - - mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) - - try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant) - except MappingDataDoesntExistException as e: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", - extra=logging_context(), - ) - raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for mapped variant {urn}: {e}") - - if not pathogenicity_statement: - logger.info( - msg="Could not construct a pathogenicity statement for this mapped variant; Variant does not have sufficient evidence to evaluate its pathogenicity.", - extra=logging_context(), - ) - raise HTTPException( - status_code=404, - detail=f"No pathogenicity statement exists for mapped variant {urn}; Variant does not have sufficient evidence to evaluate its pathogenicity.", - ) - - return pathogenicity_statement - - -@router.get( - "/vrs/{identifier}", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Fetch mapped variants by VRS identifier", -) -async def show_mapped_variants_by_identifier( - *, - identifier: Annotated[ - str, - Path( - description="String, a valid GA4GH digest based identifier.", - json_schema_extra={"example": "ga4gh:SQ.0123abcd"}, - regex=GA4GH_IR_REGEXP, - ), - ], - only_current: bool = True, - db: Session = Depends(deps.get_db), - user: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Fetch a single mapped variant by GA4GH identifier. - """ - query = select(MappedVariant).where( - or_(MappedVariant.pre_mapped["id"].astext == identifier, MappedVariant.post_mapped["id"].astext == identifier) - ) - - if only_current: - query = query.where(MappedVariant.current.is_(True)) - - items = db.scalars(query).all() - - permitted_items = [item for item in items if has_permission(user, item.variant.score_set, Action.READ).permitted] - if not permitted_items: - raise HTTPException(status_code=404, detail=f"No mapped variants with identifier {identifier} were found") - - return permitted_items - - -# for testing only -# @router.post("/map/{urn}", status_code=200, responses={404: {}, 500: {}}) -# async def map_score_set(*, urn: str, worker: ArqRedis = Depends(deps.get_worker)) -> Any: -# await worker.lpush(MAPPING_QUEUE_NAME, urn) # type: ignore -# await worker.enqueue_job( -# "variant_mapper_manager", -# None, -# None, -# None -# ) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index d7cfdfc15..78144594c 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,14 +1,25 @@ import logging from datetime import datetime -from typing import Optional +from typing import Annotated, Optional -from fastapi import APIRouter, Depends, Query, Response +from fastapi import APIRouter, Depends, Path, Query, Response from fastapi.exceptions import HTTPException +from ga4gh.core.identifiers import GA4GH_IR_REGEXP +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session from mavedb import deps +from mavedb.lib.alleles import find_variants_by_vrs_identifier +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.context import variant_annotation_context +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authentication import get_current_user from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -21,6 +32,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.variant import VariantVrsMatch from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Variants" @@ -40,6 +52,85 @@ } +def _fetch_readable_variant(db: Session, user_data: Optional[UserData], urn: str) -> Variant: + """Fetch a single variant by URN and assert the caller may read its score set. + + Raises 404 when no such variant exists, 500 when the URN is not unique (an invariant break), and the + standard 403 (via ``assert_permission``) when the score set is not readable. + """ + try: + variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + return variant + + +@router.get( + "/variants/vrs/{identifier}", + status_code=200, + response_model=list[VariantVrsMatch], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Look up variants by VRS identifier", +) +def lookup_variants_by_vrs_identifier( + *, + response: Response, + identifier: Annotated[ + str, + Path( + description="A valid GA4GH digest-based identifier for the mapped allele.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> list[VariantVrsMatch]: + """Resolve a GA4GH VRS identifier to the readable variants whose mapping links that allele. + + A deduplicated allele may be shared across score sets, so one identifier can resolve to several + variants. This is a lookup returning a collection: results are filtered to the score sets the caller + may read, and an empty list is returned when nothing readable matches. An absent identifier and a + match visible only in a private score set are deliberately indistinguishable (both yield ``[]``), so + the response never reveals a private allele's existence. + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + matches = find_variants_by_vrs_identifier(db, identifier, as_of=as_of) + permitted = [ + (variant, allele) + for variant, allele in matches + if has_permission(user_data, variant.score_set, Action.READ).permitted + ] + + return [ + VariantVrsMatch( + variant_urn=variant.urn or "", + clingen_allele_id=allele.clingen_allele_id, + vrs_id=(allele.post_mapped or {}).get("id"), + level=allele.level, + ) + for variant, allele in permitted + ] + + @router.get( "/variants/{urn}", status_code=200, @@ -72,17 +163,7 @@ def get_variant( """ save_to_logging_context({"requested_resource": urn, "as_of": as_of}) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - try: - variant = db.scalars(select(Variant).where(Variant.urn == urn)).one_or_none() - except MultipleResultsFound: - logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) - raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") - - if not variant: - logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) - raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") - - assert_permission(user_data, variant.score_set, Action.READ) + variant = _fetch_readable_variant(db, user_data, urn) # Version standing: resolve the superseding version's visibility exactly as fetch_score_set_by_urn # does — a newer version the user cannot read is not leaked (the variant then reads as current). @@ -103,3 +184,165 @@ def get_variant( visible_calibration_ids=visible_calibration_ids, as_of=as_of, ) + + +@router.get( + "/variants/{urn}/va/study-result", + status_code=200, + response_model=ExperimentalVariantFunctionalImpactStudyResult, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec StudyResult for a variant", +) +def get_variant_study_result( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> ExperimentalVariantFunctionalImpactStudyResult: + """Construct a single VA-Spec StudyResult for a variant by URN, from its mapping substrate.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, detail=f"No study result exists for variant {urn}: no mapping data exists." + ) + + try: + return variant_study_result(context) + except MappingDataDoesntExistException as e: + logger.info(msg=f"Could not construct a study result for variant {urn}: {e}", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"No study result exists for variant {urn}: {e}") + + +@router.get( + "/variants/{urn}/va/functional-statement", + status_code=200, + response_model=Statement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec functional-impact Statement for a variant", +) +def get_variant_functional_impact_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Statement: + """Construct a single VA-Spec functional-impact Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No functional impact statement exists for variant {urn}: no mapping data exists.", + ) + + try: + functional_impact = variant_functional_impact_statement(context) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a functional impact statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No functional impact statement exists for variant {urn}: {e}") + + if not functional_impact: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its functional impact.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient " + "evidence to evaluate its functional impact." + ), + ) + + return functional_impact + + +@router.get( + "/variants/{urn}/va/pathogenicity-statement", + status_code=200, + response_model=VariantPathogenicityStatement, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="Construct a VA-Spec pathogenicity Statement for a variant", +) +def get_variant_pathogenicity_statement( + *, + response: Response, + urn: str, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (Cat-VRS membership, VEP/gnomAD/ClinVar annotations) as it " + "stood at this instant, over the variant's fixed score. ISO 8601, ideally timezone-aware. " + "Content valid-time only — it never re-selects a score-set version, and scores/classifications " + "are as-of-invariant. Defaults to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> VariantPathogenicityStatement: + """Construct a single VA-Spec pathogenicity Statement for a variant by URN.""" + save_to_logging_context({"requested_resource": urn, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + variant = _fetch_readable_variant(db, user_data, urn) + + context = variant_annotation_context(db, variant, as_of=as_of) + if context is None: + raise HTTPException( + status_code=404, + detail=f"No pathogenicity statement exists for variant {urn}: no mapping data exists.", + ) + + try: + pathogenicity_statement = variant_pathogenicity_statement(context) + except MappingDataDoesntExistException as e: + logger.info( + msg=f"Could not construct a pathogenicity statement for variant {urn}: {e}", extra=logging_context() + ) + raise HTTPException(status_code=404, detail=f"No pathogenicity statement exists for variant {urn}: {e}") + + if not pathogenicity_statement: + logger.info( + msg=f"Variant {urn} does not have sufficient evidence to evaluate its pathogenicity.", + extra=logging_context(), + ) + raise HTTPException( + status_code=404, + detail=( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient " + "evidence to evaluate its pathogenicity." + ), + ) + + return pathogenicity_statement diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index e79819999..12fa5bf35 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -51,7 +51,6 @@ hgvs, job_runs, licenses, - mapped_variant, orcid, permissions, pipelines, @@ -105,7 +104,6 @@ app.include_router(job_runs.router) app.include_router(licenses.router) # app.include_router(log.router) -app.include_router(mapped_variant.router) app.include_router(orcid.router) app.include_router(permissions.router) app.include_router(pipelines.router) @@ -253,7 +251,6 @@ def customize_openapi_schema(): hgvs.metadata, licenses.metadata, # log.metadata, - mapped_variant.metadata, orcid.metadata, permissions.metadata, publication_identifiers.metadata, diff --git a/src/mavedb/view_models/variant.py b/src/mavedb/view_models/variant.py index c32abcab6..48d4edb85 100644 --- a/src/mavedb/view_models/variant.py +++ b/src/mavedb/view_models/variant.py @@ -9,6 +9,20 @@ from mavedb.view_models.mapped_variant import SavedMappedVariant +class VariantVrsMatch(BaseModel): + """One variant whose mapping links an allele bearing the queried VRS identifier. + + The ``GET /variants/vrs/{identifier}`` lookup result. The ClinGen id + level ride from the matched + (deduplicated) allele; the URN from the variant it is linked to. Replaces the legacy MappedVariant VRS + lookup on the new Allele substrate (#743, item 3.5). + """ + + variant_urn: str + clingen_allele_id: Optional[str] = None + vrs_id: Optional[str] = None + level: Optional[str] = None + + class VariantEffectMeasurementBase(BaseModel): """Properties shared by most variant effect measurement view models""" diff --git a/tests/routers/test_mapped_variants.py b/tests/routers/test_mapped_variants.py deleted file mode 100644 index 51f45452f..000000000 --- a/tests/routers/test_mapped_variants.py +++ /dev/null @@ -1,620 +0,0 @@ -# ruff: noqa: E402 - -import json - -import pytest - -from tests.helpers.util.user import change_ownership - -arq = pytest.importorskip("arq") -cdot = pytest.importorskip("cdot") -fastapi = pytest.importorskip("fastapi") - -from urllib.parse import quote_plus - -from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement -from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement -from sqlalchemy import select -from sqlalchemy.orm.session import make_transient - -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet as ScoreSetDbModel -from mavedb.models.variant import Variant -from mavedb.view_models.mapped_variant import SavedMappedVariant -from tests.helpers.constants import ( - TEST_BIORXIV_IDENTIFIER, - TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, - TEST_PUBMED_IDENTIFIER, -) -from tests.helpers.util.common import deepcamelize -from tests.helpers.util.experiment import create_experiment -from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration -from tests.helpers.util.score_set import ( - create_seq_score_set_with_mapped_variants, - create_seq_score_set_with_variants, -) - - -def test_show_mapped_variant(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["id"] == 1 - - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_when_multiple_exist(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_when_none_exists(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_show_mapped_variant_study_result(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." - - ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_study_result_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_study_result_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -def test_cannot_show_mapped_variant_study_result_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No study result exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_functional_impact_statement( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 200 - assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." - - Statement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_functional_impact_statement_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_functional_impact_statement_when_insufficient_functional_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - # insufficient evidence = no (primary) calibrations - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No functional impact statement exists for mapped variant {score_set['urn']}#1. Variant does not have sufficient evidence to evaluate its functional impact" - in response_data["detail"] - ) - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_show_mapped_variant_clinical_evidence_line( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 200 - assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] - - VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_multiple_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - session.expunge(item) - make_transient(item) - item.id = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 500 - assert response_data["detail"] == f"Multiple variants with URN {score_set['urn']}#1 were found." - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_none_exists( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert response_data["detail"] == f"Mapped variant with URN {score_set['urn']}#1 not found" - - -@pytest.mark.parametrize( - "mock_publication_fetch", - [ - [ - {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, - {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, - ] - ], - indirect=["mock_publication_fetch"], -) -def test_cannot_show_mapped_variant_clinical_evidence_line_when_no_mapping_data_exists( - client, session, data_provider, data_files, setup_router_db, mock_publication_fetch -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - create_publish_and_promote_score_calibration( - client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) - ) - - item = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert item is not None - - item.post_mapped = None - session.add(item) - session.commit() - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1: Variant {score_set['urn']}#1 does not have a post mapped variant." - in response_data["detail"] - ) - - -def test_cannot_show_mapped_variant_clinical_evidence_line_when_insufficient_pathogenicity_evidence( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - response = client.get(f"/api/v1/mapped-variants/{quote_plus(score_set['urn'] + '#1')}/va/pathogenicity-statement") - response_data = response.json() - - assert response.status_code == 404 - assert ( - f"No pathogenicity statement exists for mapped variant {score_set['urn']}#1; Variant does not have sufficient evidence to evaluate its pathogenicity" - in response_data["detail"] - ) - - -def test_show_mapped_variants_by_ga4gh_identifier(client, session, data_provider, data_files, setup_router_db): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_when_none_exist( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - fake_mapped_variant_id = mapped_variant.pre_mapped["id"] - fake_mapped_variant_id = fake_mapped_variant_id[:-3] + "aaa" # Modify the ID to ensure it doesn't exist - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(fake_mapped_variant_id)}") - assert response.status_code == 404 - - -def test_show_mapped_variants_by_ga4gh_identifier_with_non_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - # Set the mapped variant to non-current - mapped_variant.current = False - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=false" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_show_mapped_variants_by_ga4gh_identifier_with_only_current_variants( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - mapped_variant2 = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#2')) - assert mapped_variant is not None - assert mapped_variant2 is not None - - # Set the mapped variant to non-current - mapped_variant2.current = False - mapped_variant2.pre_mapped = mapped_variant.pre_mapped # Ensure both pre mapped blobs match besides current status - session.add(mapped_variant) - session.commit() - - response = client.get( - f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}?only_current=true" - ) - response_data = response.json() - - assert response.status_code == 200 - assert len(response_data) == score_set["numVariants"] - 1 - - for response_data in response_data: - assert response_data["preMapped"]["id"] == mapped_variant.pre_mapped["id"] - SavedMappedVariant.model_validate_json(json.dumps(response_data)) - - -def test_cannot_show_mapped_variants_by_ga4gh_identifier_without_score_set_permissions( - client, session, data_provider, data_files, setup_router_db -): - experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( - client, - session, - data_provider, - experiment["urn"], - data_files / "scores.csv", - ) - - change_ownership(session, score_set["urn"], ScoreSetDbModel) - - mapped_variant = session.scalar(select(MappedVariant).join(Variant).where(Variant.urn == f'{score_set["urn"]}#1')) - assert mapped_variant is not None - - response = client.get(f"/api/v1/mapped-variants/vrs/{quote_plus(mapped_variant.pre_mapped['id'])}") - assert response.status_code == 404 diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index ea1c01086..dae7fd50e 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -54,7 +54,8 @@ def _post_mapped() -> dict: def _seed_mapping(session, variant_urn): """Give a variant a live coding-measured mapping record whose authoritative allele carries a digest + ClinGen id + a live VEP consequence, its genomic projection sibling (shared - projection_group), and a protein apex — enough to build Cat-VRS and exercise the projection axes.""" + projection_group), a protein apex, and a synonymous cousin (a nt encoder of the same consequence in + a different projection group) — enough to build Cat-VRS and exercise the projection + convergent axes.""" variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == variant_urn)) record = MappingRecord( variant_id=variant.id, @@ -78,13 +79,16 @@ def _seed_mapping(session, variant_urn): protein = Allele( vrs_digest="prot-digest", level="protein", post_mapped=_post_mapped(), hgvs_p="NP_000537.3:p.Ala406Thr" ) - session.add_all([measured, genomic, protein]) + cousin = Allele( + vrs_digest="cousin-digest", level="cdna", post_mapped=_post_mapped(), hgvs_c="NM_000546.6:c.1218C>T" + ) + session.add_all([measured, genomic, protein, cousin]) session.commit() session.add_all( [ # The measured cdna link and its genomic projection share a projection_group; the protein - # apex is in no pair (group None). + # apex is in no pair (group None); the cousin sits in its own projection group. MappingRecordAllele( mapping_record_id=record.id, allele_id=measured.id, is_authoritative=True, projection_group=0 ), @@ -92,6 +96,9 @@ def _seed_mapping(session, variant_urn): mapping_record_id=record.id, allele_id=genomic.id, is_authoritative=False, projection_group=0 ), MappingRecordAllele(mapping_record_id=record.id, allele_id=protein.id, is_authoritative=False), + MappingRecordAllele( + mapping_record_id=record.id, allele_id=cousin.id, is_authoritative=False, projection_group=1 + ), VepAlleleConsequence( allele_id=measured.id, functional_consequence="missense_variant", @@ -125,6 +132,9 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["mode"] == "projection" # Spec-pure Cat-VRS carries its own field names (no camelization of the nested GA4GH object). assert body["molecularRepresentation"]["type"] == "CategoricalVariant" + # State A: the full-closure Cat-VRS member set agrees with the sidecar (every linked allele is a + # member, the cousin included), so the envelope is internally consistent. + assert len(body["molecularRepresentation"]["members"]) == len(body["alleles"]) # The alleles identity sidecar serializes camelCase, keyed by digest; the protein member is a # translation of the defining coding allele (which itself has no relation to itself). assert body["alleles"]["cdna-digest"]["level"] == "cdna" @@ -142,6 +152,10 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). assert body["alleles"]["prot-digest"]["derivation"] == "projection" assert "projectionOf" not in body["alleles"]["prot-digest"] + # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is + # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). + assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" + assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True assert "supersededByScoreSet" not in body # dropped by exclude_none when current diff --git a/tests/routers/test_variant_annotation.py b/tests/routers/test_variant_annotation.py new file mode 100644 index 000000000..54d7fadf8 --- /dev/null +++ b/tests/routers/test_variant_annotation.py @@ -0,0 +1,352 @@ +# ruff: noqa: E402 +"""Router tests for the variant annotation surface relocated onto the Allele substrate. + +The VA-Spec statement endpoints (``/variants/{urn}/va/*``) and the VRS-identifier lookup +(``/variants/vrs/{identifier}``) moved off the legacy ``/mapped-variants`` router in #743; they build +from the ``MappingRecord`` / ``Allele`` substrate (seeded by ``seed_annotation_substrate``), never the +legacy ``MappedVariant``. +""" + +import json +from datetime import datetime, timezone + +import pytest + +from tests.helpers.util.user import change_ownership + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote_plus + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +from sqlalchemy import select + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.models.variant import Variant as VariantDbModel +from tests.helpers.constants import ( + TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, + TEST_GA4GH_IDENTIFIER, + TEST_PUBMED_IDENTIFIER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.common import deepcamelize +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import create_publish_and_promote_score_calibration +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + create_seq_score_set_with_variants, + seed_annotation_substrate, +) + +_PUBLICATION_FETCH = [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER}, +] + + +# --- StudyResult ----------------------------------------------------------------------------------- + + +def test_variant_study_result(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant effect study result for {score_set['urn']}#1." + ExperimentalVariantFunctionalImpactStudyResult.model_validate_json(json.dumps(response_data)) + + +def test_variant_study_result_404_when_variant_missing(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + urn = f"{score_set['urn']}#404" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert response.json()["detail"] == f"variant with URN '{urn}' not found" + + +def test_variant_study_result_404_when_no_mapping_data(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + # The variant exists but has no mapping record seeded on the new substrate. + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/study-result") + + assert response.status_code == 404 + assert f"No study result exists for variant {urn}: no mapping data exists." in response.json()["detail"] + + +# --- Functional-impact Statement ------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_functional_impact_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/functional-statement") + response_data = response.json() + + assert response.status_code == 200 + assert response_data["description"] == f"Variant functional impact statement for {score_set['urn']}#1." + Statement.model_validate_json(json.dumps(response_data)) + + +def test_variant_functional_impact_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # Mapped on the new substrate, but no (primary) calibration => insufficient evidence. + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/functional-statement") + + assert response.status_code == 404 + assert ( + f"No functional impact statement exists for variant {urn}. Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- Pathogenicity Statement ----------------------------------------------------------------------- + + +@pytest.mark.parametrize("mock_publication_fetch", [_PUBLICATION_FETCH], indirect=["mock_publication_fetch"]) +def test_variant_pathogenicity_statement( + client, session, data_provider, data_files, setup_router_db, mock_publication_fetch +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_publish_and_promote_score_calibration( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + seed_annotation_substrate(session, score_set) + + response = client.get(f"/api/v1/variants/{quote_plus(score_set['urn'] + '#2')}/va/pathogenicity-statement") + response_data = response.json() + + assert response.status_code == 200 + assert f"Variant pathogenicity statement for {score_set['urn']}#2" in response_data["description"] + VariantPathogenicityStatement.model_validate_json(json.dumps(response_data)) + + +def test_variant_pathogenicity_statement_404_when_insufficient_evidence( + client, session, data_provider, data_files, setup_router_db +): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/pathogenicity-statement") + + assert response.status_code == 404 + assert ( + f"No pathogenicity statement exists for variant {urn}; Variant does not have sufficient evidence" + in response.json()["detail"] + ) + + +# --- VRS-identifier lookup ------------------------------------------------------------------------- + + +def test_lookup_variants_by_vrs_identifier(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # vrs_digest stores the full GA4GH VRS id in production (the mapper writes post_mapped["id"] into it), + # so the lookup matches the identifier against vrs_digest. The allele's ClinGen id rides alongside the URN. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + response_data = response.json() + + assert response.status_code == 200 + assert len(response_data) == 1 + assert response_data[0]["variantUrn"] == f"{score_set['urn']}#1" + assert response_data[0]["clingenAlleleId"] == "CA1" + assert response_data[0]["vrsId"] == TEST_GA4GH_IDENTIFIER + + +def test_lookup_variants_by_vrs_identifier_empty_when_none_match( + client, session, data_provider, data_files, setup_router_db +): + """The lookup is collection-shaped: no match is an empty list, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + # A well-formed GA4GH id (32-char digest) that no seeded allele carries. + response = client.get(f"/api/v1/variants/vrs/{quote_plus('ga4gh:VA.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA')}") + + assert response.status_code == 200 + assert response.json() == [] + + +def test_lookup_variants_by_vrs_identifier_empty_without_read_permission( + client, session, data_provider, data_files, setup_router_db +): + """Existence-hiding: an identifier that matches only a private score set is filtered to an empty list, + indistinguishable from a genuine no-match — so the response never reveals the private allele exists.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + # The identifier matches a real allele (by vrs_digest); the permission filter drops it to an empty list. + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[AlleleSpec(digest=TEST_GA4GH_IDENTIFIER, level="cdna", is_authoritative=True)], + ) + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + response = client.get(f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}") + + assert response.status_code == 200 + assert response.json() == [] + + +# --- as_of temporal behavior ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("va_endpoint", ["study-result", "functional-statement", "pathogenicity-statement"]) +def test_va_endpoint_404_when_no_mapping_data_at_as_of( + client, session, data_provider, data_files, setup_router_db, va_endpoint +): + """A past instant with no live mapping yields the single-resource 404 — the statement did not exist then. + + ``as_of`` is threaded into ``variant_annotation_context``, so a far-past instant sees no live mapping + record and the derived VA statement cannot be constructed. This is the deliberate contrast with the + score-set *collection* endpoints: an empty collection is 200 + empty, but a single derived resource + that does not exist at the requested instant is a 404. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + urn = f"{score_set['urn']}#1" + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + response = client.get(f"/api/v1/variants/{quote_plus(urn)}/va/{va_endpoint}", params={"as_of": past.isoformat()}) + + assert response.status_code == 404 + assert "no mapping data exists" in response.json()["detail"] + + +def test_variant_study_result_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The study-result route reconstructs its subject at ``as_of`` and echoes the instant on ``X-As-Of``. + + Study result needs only live mapping (no calibration), so it cleanly exercises the positive path: a + far-future instant still sees the freshly-seeded substrate as live and returns 200. Omitting ``as_of`` + reports ``current``. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + + url = f"/api/v1/variants/{quote_plus(score_set['urn'] + '#1')}/va/study-result" + + # No as_of: the header reports the current standing. + current_response = client.get(url) + assert current_response.status_code == 200 + assert current_response.headers["X-As-Of"] == "current" + + # Far future: the seeded mapping is live, so the subject reconstructs and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + +def test_lookup_variants_by_vrs_identifier_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """The VRS lookup filters its matches by ``as_of`` (the allele link must be live at that instant).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant = session.scalar(select(VariantDbModel).where(VariantDbModel.urn == f"{score_set['urn']}#1")) + seed_mapping_record( + session, + variant, + assay_level="cdna", + alleles=[ + AlleleSpec( + digest=TEST_GA4GH_IDENTIFIER, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + ) + ], + ) + url = f"/api/v1/variants/vrs/{quote_plus(TEST_GA4GH_IDENTIFIER)}" + + # Far future: the allele link is live, so the identifier resolves and the instant echoes back. + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert len(future_response.json()) == 1 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + + # Far past: nothing was live yet, so no allele matches — an empty collection, not a 404. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.json() == [] From 275a75a4a3c50ccea0a3ef7d188d97e59359ce83 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Sat, 18 Jul 2026 13:24:50 -0700 Subject: [PATCH 80/93] feat(alleles): add allele-detail endpoint (digest / CAID / PAID) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET /alleles/{identifier}, the allele-grain sibling of GET /variants/{urn}: anchor identity + the full cross-layer equivalence class (each member labelled relative to the focus — projection / candidate / convergent) + digest-keyed annotations. One path overloaded across a VRS digest, a nucleotide CAID, or a protein PAID via a custom Starlette convertor. Public and measurement-agnostic; no Cat-VRS (that stays rooted at the measured allele on the variant view). Unify the molecular core with variant detail: extract a shared AlleleIdentity (lib + view model) used by both endpoints' `alleles` map, and mark the anchored allele with `isFocus` — replacing the `derivation: "authoritative"` value on variant detail (there is no authoritative *variant* at the allele grain). Factor the nucleotide-level set into SequenceLevel.NUCLEOTIDE_LEVELS, shared by cat_vrs and allele_detail. --- src/mavedb/lib/allele_detail.py | 188 ++++++++++++++++ src/mavedb/lib/allele_identity.py | 68 ++++++ src/mavedb/lib/cat_vrs.py | 11 +- src/mavedb/lib/variant_detail.py | 78 +------ src/mavedb/models/enums/sequence_level.py | 14 ++ src/mavedb/routers/alleles.py | 129 +++++++++++ src/mavedb/server_main.py | 3 + src/mavedb/view_models/allele_annotation.py | 6 +- src/mavedb/view_models/allele_detail.py | 39 ++++ src/mavedb/view_models/allele_identity.py | 29 +++ src/mavedb/view_models/variant_detail.py | 29 +-- tests/lib/test_allele_detail.py | 203 ++++++++++++++++++ tests/lib/test_variant_detail.py | 21 +- tests/routers/test_allele.py | 224 ++++++++++++++++++++ tests/routers/test_variant.py | 8 +- 15 files changed, 933 insertions(+), 117 deletions(-) create mode 100644 src/mavedb/lib/allele_detail.py create mode 100644 src/mavedb/lib/allele_identity.py create mode 100644 src/mavedb/routers/alleles.py create mode 100644 src/mavedb/view_models/allele_detail.py create mode 100644 src/mavedb/view_models/allele_identity.py create mode 100644 tests/lib/test_allele_detail.py create mode 100644 tests/routers/test_allele.py diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py new file mode 100644 index 000000000..14603f509 --- /dev/null +++ b/src/mavedb/lib/allele_detail.py @@ -0,0 +1,188 @@ +"""The allele-detail view backing ``GET /alleles/{digest:CAID}``. + +The allele-grain sibling of :mod:`lib.variant_detail`. Where the variant view anchors on a +*measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) +and serves that allele's identity, its full cross-layer **equivalence class**, and the digest-keyed +annotations that ride alongside. + +It is measurement-agnostic, and that shapes what it does *not* carry: + +- No score, counts, classification, or version standing — those are properties of a measured variant, + not of an allele (an allele is deduplicated by digest and shared across every score set mapping to it). +- No spec-pure Cat-VRS. The ``CategoricalVariant`` / ``DefiningAlleleConstraint`` is emitted only by the + measurement view, rooted at the measured allele. Here the members are labelled on a flat map + *relative to the focus allele*. + +**The equivalence class is the full cross-record union** (:func:`lib.alleles.get_allele_translations`): +every allele co-linked to any live record touching the focus. That is the discoverable "everything +related to this change" set. Each member is then labelled relative to the focus — which the graph +determines even without a measurement, because candidate-ness is *directional* (protein→nt is +many-to-one ⇒ candidate; nt→protein is deterministic ⇒ consequence; nt↔nt is a faithful coordinate +transform). "Less power than a measurement" only means we cannot say which nt was *measured* — but we +can still say faithful vs. candidate vs. convergent from the graph and the focus. + +``as_of`` reconstructs the molecular layer — class membership + annotations — at the past instant. The +focus allele's own identity is immutable and content-addressed. ``as_of`` defaults to currently-live rows. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity +from mavedb.lib.alleles import get_allele_translations +from mavedb.lib.cat_vrs import CatVrsRelation +from mavedb.lib.vrs import vrs_object_from_mapped_variant +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel +from mavedb.models.mapping_record_allele import MappingRecordAllele + + +@dataclass(frozen=True) +class AlleleDetail: + """The assembled allele-detail envelope (transit; serialized by ``view_models.allele_detail``).""" + + # Flat anchor identity — the queried allele's own molecular facts (for a CAID fetch, its + # representative allele; ``clingenAlleleId`` carries the CAID either way). + digest: str + level: Optional[str] + hgvs: Optional[str] # reference-frame HGVS (exactly one of hgvs_g/c/p, depending on level) + clingen_allele_id: Optional[str] + vrs: Optional[dict[str, Any]] # spec-pure GA4GH VRS variation from post_mapped + + # The full cross-layer equivalence class, keyed by VRS digest and sharing keys with `annotations`. + # Each member is an `AlleleIdentity` labelled relative to the focus. The focus allele(s) carry + # `is_focus=True`. + alleles: dict[str, AlleleIdentity] + + # External annotations, keyed by VRS digest, for every member of the class. + annotations: dict[str, AlleleAnnotations] + + +def _focus_projection_pairs(db: Session, focus_alleles: list[Allele], *, as_of: Optional[datetime]) -> dict[str, str]: + """A symmetric ``digest -> sibling digest`` map for each focus allele's c↔g coordinate partner. + + Read from any one live record of the focus: the partner is the ≤1 *other* allele sharing the focus + allele's ``projection_group`` (a projection group is a c↔g pair). This is the one place the + focus/partner-vs-convergent-cousin distinction needs the link-level ``projection_group`` — every + other member is classified by level alone. + """ + pairs: dict[str, str] = {} + for focus in focus_alleles: + if focus.vrs_digest is None: + continue + link = db.scalar( + select(MappingRecordAllele) + .where(MappingRecordAllele.allele_id == focus.id) + .where(MappingRecordAllele.projection_group.isnot(None)) + .where(MappingRecordAllele.live_at(as_of)) + .limit(1) + ) + if link is None: + continue + sibling_digests = db.scalars( + select(Allele.vrs_digest) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where(MappingRecordAllele.mapping_record_id == link.mapping_record_id) + .where(MappingRecordAllele.projection_group == link.projection_group) + .where(MappingRecordAllele.allele_id != focus.id) + .where(MappingRecordAllele.live_at(as_of)) + ).all() + for sibling in sibling_digests: + if sibling is not None: + pairs[focus.vrs_digest] = sibling + pairs.setdefault(sibling, focus.vrs_digest) + return pairs + + +def _member_label( + *, focus_is_protein: bool, member_level: Optional[str], is_coordinate_partner: bool +) -> tuple[Optional[CatVrsRelation], Optional[AlleleDerivation]]: + """The (relation, derivation) for a non-focus member, read *relative to the focus*. + + - **protein focus** → every nt member is a reverse-translation ``candidate`` that ``encodes`` it. + - **nt focus** → the protein consequence is ``translation_of`` (a deterministic ``projection``); the + focus's coordinate partner is ``coordinate_representation_of`` (also ``projection``); any other nt + is a synonymous cousin — ``co_encodes`` / ``convergent`` (a distinct change sharing the consequence). + """ + if focus_is_protein: + if member_level in NUCLEOTIDE_LEVELS: + return CatVrsRelation.ENCODES, AlleleDerivation.CANDIDATE + return None, None + + if member_level == SequenceLevel.protein.value: + return CatVrsRelation.TRANSLATION_OF, AlleleDerivation.PROJECTION + if member_level in NUCLEOTIDE_LEVELS: + if is_coordinate_partner: + return CatVrsRelation.COORDINATE_REPRESENTATION_OF, AlleleDerivation.PROJECTION + return CatVrsRelation.CO_ENCODES, AlleleDerivation.CONVERGENT + return None, None + + +def get_allele_detail( + db: Session, anchor: Allele, *, focus_digests: set[str], as_of: Optional[datetime] = None +) -> AlleleDetail: + """Assemble the allele-detail envelope anchored on ``anchor``. + + ``focus_digests`` is the set of VRS digests that are the *focus* of this view — a single digest for + a by-digest fetch, or the (genomic + coding) representations of a CAID for a by-CAID fetch. The + equivalence class is the record-agnostic union co-linked to any live record touching ``anchor`` + (:func:`get_allele_translations`, which includes ``anchor``); an orphan allele with no live links + falls back to itself. Every member is labelled relative to the focus (see :func:`_member_label`). + ``as_of`` reconstructs the molecular layer; see the module docstring. + """ + equivalence_class = get_allele_translations(db, anchor.id, as_of=as_of) or [anchor] + + annotations = get_allele_annotations(db, equivalence_class, as_of=as_of) + + focus_alleles = [a for a in equivalence_class if a.vrs_digest in focus_digests] or [anchor] + focus_is_protein = any(a.level == SequenceLevel.protein.value for a in focus_alleles) + + pairs = _focus_projection_pairs(db, focus_alleles, as_of=as_of) + # The focus's coordinate partner(s) that are not themselves focus (for a CAID fetch the c↔g pair is + # jointly the focus, so this is empty and the partners are simply flagged is_focus instead). + coordinate_partner_digests = {pairs[d] for d in focus_digests if d in pairs} - focus_digests + + alleles: dict[str, AlleleIdentity] = {} + for member in equivalence_class: + digest = member.vrs_digest + if digest is None: + continue + + if digest in focus_digests: + relation, derivation = None, None + else: + rel, der = _member_label( + focus_is_protein=focus_is_protein, + member_level=member.level, + is_coordinate_partner=digest in coordinate_partner_digests, + ) + relation = rel.value if rel is not None else None + derivation = der.value if der is not None else None + + alleles[digest] = AlleleIdentity( + level=member.level, + hgvs=member.hgvs_g or member.hgvs_c or member.hgvs_p, + clingen_allele_id=member.clingen_allele_id, + is_focus=digest in focus_digests, + relation=relation, + derivation=derivation, + projection_of=pairs.get(digest), + ) + + return AlleleDetail( + digest=anchor.vrs_digest or "", + level=anchor.level, + hgvs=anchor.hgvs_g or anchor.hgvs_c or anchor.hgvs_p, + clingen_allele_id=anchor.clingen_allele_id, + vrs=vrs_object_from_mapped_variant(anchor.post_mapped).model_dump(mode="json", exclude_none=True) + if anchor.post_mapped is not None + else None, + alleles=alleles, + annotations=annotations, + ) diff --git a/src/mavedb/lib/allele_identity.py b/src/mavedb/lib/allele_identity.py new file mode 100644 index 000000000..7f4c12a74 --- /dev/null +++ b/src/mavedb/lib/allele_identity.py @@ -0,0 +1,68 @@ +"""The MaveDB molecular-identity view of an allele — shared by variant detail and allele detail. + +Both serving views present a set of alleles keyed by VRS digest, each labelled by how it relates to +the allele the view is anchored on (its *focus*). The two views differ only in what the focus is: +``GET /variants/{urn}`` focuses the **measured** allele, ``GET /alleles/{digest}`` focuses the +**queried** allele. The identity shape is shared and its labels are always read *relative to the +focus*. + +This module owns that shared shape (:class:`AlleleIdentity`) and its provenance vocabulary +(:class:`AlleleDerivation`). The Cat-VRS structural relation codes live in :mod:`lib.cat_vrs` +(``CatVrsRelation``); this module carries only the derivation (confidence/provenance) axis, which is +orthogonal. +""" + +from dataclasses import dataclass +from enum import Enum +from typing import Optional + + +class AlleleDerivation(str, Enum): + """How an allele's representation was arrived at, *relative to the focus allele*: its + confidence/provenance axis. Orthogonal to the Cat-VRS structural ``relation`` axis: ``relation`` + says which level relates to which, ``derivation`` says how much to trust the representation. + + There is deliberately **no** ``authoritative`` value: the focus allele is marked by + :attr:`AlleleIdentity.is_focus`, not by a derivation. That keeps the axis meaningful without a + measurement — on the allele view there is no *measured* allele, only the queried focus. + """ + + # Deterministic and precise, given (assembly, transcript): the focus's coordinate partner + # (nucleotide↔nucleotide) and its protein consequence (nucleotide→protein). + PROJECTION = "projection" + # Reverse-translation of a *protein* focus: genuinely ambiguous. One member of the fanned-out + # equivalence class, not a precise and deterministic pair. + CANDIDATE = "candidate" + # A distinct, precisely-known nucleotide change that *converges* on the same protein consequence + # a different codon produces. Not ambiguous like a candidate nor a projection *of* the focus. A + # separate, unmeasured variant that simply shares the consequence. + CONVERGENT = "convergent" + + +@dataclass(frozen=True) +class AlleleIdentity: + """The MaveDB molecular-identity facts for one allele in a view's ``alleles`` map, keyed by VRS + digest. ``level`` + reference-frame ``hgvs`` (exactly one of the allele's genomic/coding/protein + columns) are what the UI labels by — never the digest. + + Four axes, all read *relative to the view's focus allele*: + + - ``is_focus``: this allele is the one the view is anchored on (the measured allele on variant + detail; the queried allele/CAID on allele detail). Its ``relation``/``derivation`` are ``None`` — + it is the reference point the others are described against. + - ``relation`` (Cat-VRS, structural): member→focus relation (``coordinate_representation_of`` / + ``translation_of`` / ``encodes`` / ``co_encodes``). ``None`` for the focus itself. + - ``derivation`` (provenance): :class:`AlleleDerivation` — projection / candidate / convergent. + Orthogonal to ``relation``; never conflate them. + - ``projection_of`` (provenance): the VRS digest of this allele's c↔g projection sibling, when + known. ``None`` for the protein apex, pre-reverse-translation data, or where the pairing is not + resolved in this view. + """ + + level: Optional[str] + hgvs: Optional[str] + clingen_allele_id: Optional[str] + is_focus: bool + relation: Optional[str] + derivation: Optional[str] + projection_of: Optional[str] diff --git a/src/mavedb/lib/cat_vrs.py b/src/mavedb/lib/cat_vrs.py index b9ad6d39c..02e1d1d03 100644 --- a/src/mavedb/lib/cat_vrs.py +++ b/src/mavedb/lib/cat_vrs.py @@ -31,7 +31,7 @@ from mavedb.lib.logging.context import logging_context from mavedb.lib.vrs import vrs_object_from_mapped_variant from mavedb.models.allele import Allele -from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.enums.sequence_level import NUCLEOTIDE_LEVELS, SequenceLevel from mavedb.models.mapping_record_allele import MappingRecordAllele logger = logging.getLogger(__name__) @@ -63,9 +63,6 @@ class CatVrsMode(str, Enum): REVERSE_TRANSLATION = "reverse_translation" # Mode 2 — protein measured; score is implied. -_NUCLEOTIDE_LEVELS = {SequenceLevel.genomic.value, SequenceLevel.cdna.value} - - @dataclass class CategoricalVariantTransit: """The spec-pure Cat-VRS object plus the MaveDB layer that rides beside it.""" @@ -87,7 +84,7 @@ def is_convergent_cousin( - The member is in a different projection group than the defining (measured) allele. """ return ( - member_level in _NUCLEOTIDE_LEVELS + member_level in NUCLEOTIDE_LEVELS and member_group is not None and defining_group is not None and member_group != defining_group @@ -109,12 +106,12 @@ def _relation_for( """ # Protein measured. Every nt member encodes it, the protein member is the defining. Grouping is irrelevant. if defining_level == SequenceLevel.protein.value: - return CatVrsRelation.ENCODES if member_level in _NUCLEOTIDE_LEVELS else None + return CatVrsRelation.ENCODES if member_level in NUCLEOTIDE_LEVELS else None # Nt measured (genomic or cdna). if member_level == SequenceLevel.protein.value: return CatVrsRelation.TRANSLATION_OF - if member_level in _NUCLEOTIDE_LEVELS: + if member_level in NUCLEOTIDE_LEVELS: if is_convergent_cousin(member_level, member_group, defining_group=defining_group): return CatVrsRelation.CO_ENCODES return CatVrsRelation.COORDINATE_REPRESENTATION_OF diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 8f06c7cc5..567fcf226 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -18,13 +18,13 @@ from dataclasses import dataclass from datetime import datetime -from enum import Enum from typing import Any, Optional from sqlalchemy import select from sqlalchemy.orm import Session from mavedb.lib.allele_annotations import AlleleAnnotations, get_allele_annotations +from mavedb.lib.allele_identity import AlleleDerivation, AlleleIdentity from mavedb.lib.alleles import get_live_record_allele_links from mavedb.lib.cat_vrs import categorical_variant_for_variant, is_convergent_cousin from mavedb.lib.score_calibrations import calibration_preference_key @@ -54,63 +54,6 @@ class VariantClassificationRecord: classification: ScoreCalibrationFunctionalClassification -class AlleleDerivation(str, Enum): - """How an allele's representation was arrived at — its confidence/provenance axis. - - Distinct from (orthogonal to) the Cat-VRS ``relation`` axis: ``relation`` is *structural* (which - level relates to which, member→defining), whereas ``derivation`` is *epistemic* (how much to trust - the representation). See the design's "Semantics note — two axes, keep them separate". Derived at - serialization time from ``is_authoritative`` + the assay level — no stored column. - """ - - # The assay's actual measurement (the authoritative link). Precise by definition. - AUTHORITATIVE = "authoritative" - # Deterministic and precise. Derived from the measured change itself: nucleotide↔nucleotide (its - # coordinate partner) and nucleotide→protein (its consequence) are both deterministic given - # (assembly, transcript). - PROJECTION = "projection" - # Reverse-translation output of a *protein* measurement — genuinely ambiguous (many synonymous - # codons). One member of the fanned-out equivalence class, not a precise coordinate. - CANDIDATE = "candidate" - # A distinct, precisely-known nucleotide change that *converges* on the measured protein consequence - # from a different codon (a synonymous cousin under a nucleotide assay). Not ambiguous like a - # candidate, and not a projection *of* the measured change — a separate, unmeasured variant that - # merely shares the consequence. Pairs with the ``co_encodes`` Cat-VRS relation. - CONVERGENT = "convergent" - - -@dataclass(frozen=True) -class AlleleIdentity: - """The MaveDB molecular-identity facts for one of the variant's linked alleles. - - Rides alongside the spec-pure Cat-VRS keyed by VRS digest (the ``alleles`` map). ``level`` and - ``hgvs`` (the reference-frame HGVS, exactly one of the allele's genomic/coding/protein columns) are - what the UI labels the per-level annotation panel by — *never* the digest. - - Three axes ride here, deliberately independent: - - - ``relation`` (Cat-VRS, structural): this allele's member→defining relation - (``is_protein_of`` / ``coordinate_representation_of`` / …). ``None`` when it *is* the measured - allele, or when the allele is not a Cat-VRS member. Sourced from - ``cat_vrs.CategoricalVariantTransit.member_relations``. - - ``derivation`` (provenance): :class:`AlleleDerivation` — authoritative / projection / candidate / - convergent. Orthogonal to ``relation``; never conflate the two (a protein member of a nucleotide - assay has ``relation=translation_of`` but ``derivation=projection``; a synonymous cousin has - ``relation=co_encodes`` and ``derivation=convergent``; the nucleotide fan-out of a protein assay is - ``derivation=candidate``). - - ``projection_of`` (provenance): the VRS digest of this allele's projection sibling — the ≤1 *other* - member of its ``projection_group`` (a c↔g pair). ``None`` for the protein apex (group ``NULL``), - for pre-reverse-translation data, and where a level's projection failed. - """ - - level: Optional[str] - hgvs: Optional[str] - clingen_allele_id: Optional[str] - relation: Optional[str] - derivation: Optional[str] - projection_of: Optional[str] - - @dataclass(frozen=True) class VariantDetail: """The assembled variant-detail envelope (transit; serialized by ``view_models.variant``).""" @@ -177,13 +120,11 @@ def _classifications_for_variant( ] -def _derivation_for( - *, is_authoritative: bool, assay_level: Optional[SequenceLevel], is_cousin: bool -) -> AlleleDerivation: - """The provenance of a linked allele's representation, from ``is_authoritative`` + the assay level. +def _derivation_for(*, assay_level: Optional[SequenceLevel], is_cousin: bool) -> AlleleDerivation: + """The provenance of a *non-focus* linked allele's representation, from the assay level. - The measured allele is ``authoritative``. Every *other* allele's provenance turns on the protein → - codon boundary, which surfaces three ways: + The measured allele is the focus (``is_focus=True``, no derivation). Every *other* allele's + provenance turns on the protein → codon boundary, which surfaces three ways: - **protein assay** (``assay_level`` protein) → ``candidate`` for the whole derived (nucleotide) fan-out: reverse translation is genuinely ambiguous (which codon was measured is unknown). @@ -199,8 +140,6 @@ def _derivation_for( ``coordinate_representation_of`` / ``translation_of`` ↔ ``projection``, protein-assay ``encodes`` ↔ ``candidate``. """ - if is_authoritative: - return AlleleDerivation.AUTHORITATIVE if assay_level == SequenceLevel.protein.value: return AlleleDerivation.CANDIDATE if is_cousin: @@ -301,10 +240,11 @@ def get_variant_detail( level=allele.level, hgvs=allele.hgvs_g or allele.hgvs_c or allele.hgvs_p, clingen_allele_id=allele.clingen_allele_id, + is_focus=link.is_authoritative, relation=relation.value if relation is not None else None, - derivation=_derivation_for( - is_authoritative=link.is_authoritative, assay_level=assay_level, is_cousin=is_cousin - ).value, + derivation=( + None if link.is_authoritative else _derivation_for(assay_level=assay_level, is_cousin=is_cousin).value + ), projection_of=projection_of, ) diff --git a/src/mavedb/models/enums/sequence_level.py b/src/mavedb/models/enums/sequence_level.py index 016c36f96..62875f99f 100644 --- a/src/mavedb/models/enums/sequence_level.py +++ b/src/mavedb/models/enums/sequence_level.py @@ -25,6 +25,16 @@ def from_wire(cls, code: str) -> "SequenceLevel": except KeyError as exc: raise ValueError(f"Unknown sequence level wire code: {code!r}") from exc + @classmethod + def nucleotide_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are nucleotide (genomic or coding).""" + return {cls.genomic, cls.cdna} + + @classmethod + def protein_levels(cls) -> set["SequenceLevel"]: + """Return the subset of levels that are protein (protein only).""" + return {cls.protein} + # Module-level so it doesn't get mistaken for an enum member. _WIRE_TO_LEVEL: dict[str, SequenceLevel] = { @@ -32,3 +42,7 @@ def from_wire(cls, code: str) -> "SequenceLevel": "c": SequenceLevel.cdna, "g": SequenceLevel.genomic, } + + +NUCLEOTIDE_LEVELS = {level.value for level in SequenceLevel.nucleotide_levels()} +PROTEIN_LEVELS = {level.value for level in SequenceLevel.protein_levels()} diff --git a/src/mavedb/routers/alleles.py b/src/mavedb/routers/alleles.py new file mode 100644 index 000000000..abd575718 --- /dev/null +++ b/src/mavedb/routers/alleles.py @@ -0,0 +1,129 @@ +import logging +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Path, Query, Response +from fastapi.exceptions import HTTPException +from sqlalchemy import select +from sqlalchemy.orm import Session +from starlette.convertors import Convertor, register_url_convertor + +from mavedb import deps +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.lib.logging import LoggedRoute +from mavedb.lib.logging.context import save_to_logging_context +from mavedb.models.allele import Allele +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.routers.shared import ( + ACCESS_CONTROL_ERROR_RESPONSES, + PUBLIC_ERROR_RESPONSES, + ROUTER_BASE_PREFIX, +) +from mavedb.view_models.allele_detail import AlleleDetail + +TAG_NAME = "Alleles" + +logger = logging.getLogger(__name__) + + +# One resource, three identifier forms — a GA4GH VRS digest (one allele), a ClinGen nucleotide id (CAID, +# the nt-canonical change) or protein id (PAID). Rather than split the path or add a query param, overload +# a single `/alleles/{identifier}` with a custom Starlette convertor (the same pattern the publication +# router uses for DOI/PubMed/…). The convertor matches any of the three forms; the handler dispatches by +# form. In the OpenAPI spec this is still just a string path parameter. +class AlleleIdentifierConverter(Convertor): + # A GA4GH IR (ga4gh:.<32 base64url>) OR a ClinGen allele id (CA…/PA… + digits). + # NOTE: Multivariant CAIDs (CA…/PA… + digits + `,` + CA…/PA… + digits) are intentionally excluded. + regex = r"(?:ga4gh:[^.]+\.[0-9A-Za-z_\-]{32})|(?:[CP]A[0-9]+)" + + def convert(self, value: str) -> str: + return value + + def to_string(self, value: str) -> str: + return str(value) + + +register_url_convertor("allele_identifier", AlleleIdentifierConverter()) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}", + tags=[TAG_NAME], + responses={**PUBLIC_ERROR_RESPONSES}, + route_class=LoggedRoute, +) + +metadata = { + "name": TAG_NAME, + "description": "Retrieve deduplicated alleles — identity, cross-layer equivalence class, and " + "external annotations — by VRS digest or ClinGen allele id (CAID / PAID).", +} + + +def _representative(alleles: list[Allele]) -> Allele: + """The allele whose identity fills the flat anchor fields for a ClinGen-id fetch. A CAID names one nt + change in up to two frames (genomic + coding) — prefer the coding frame, then genomic; a PAID matches + the single protein allele, which falls through to ``alleles[0]``.""" + by_level = {a.level: a for a in alleles} + return by_level.get(SequenceLevel.cdna.value) or by_level.get(SequenceLevel.genomic.value) or alleles[0] + + +@router.get( + "/alleles/{identifier:allele_identifier}", + status_code=200, + response_model=AlleleDetail, + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + response_model_exclude_none=True, + summary="Fetch allele detail by VRS digest, CAID, or PAID", +) +def get_allele( + *, + response: Response, + identifier: str = Path( + description="A GA4GH VRS digest (one allele), or a ClinGen allele id — a nucleotide CAID (the " + "nt-canonical change, its genomic + coding frames) or a protein PAID.", + json_schema_extra={"example": "ga4gh:VA.0123abcd"}, + ), + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (equivalence-class membership + VEP/gnomAD/ClinVar " + "annotations) as it stood at this instant. ISO 8601, ideally timezone-aware. The focus " + "allele's own identity is content-addressed and immutable, so it is unaffected. Defaults to " + "current." + ), + ), + db: Session = Depends(deps.get_db), +): + """Fetch the detail envelope for a deduplicated allele, by any of its identifiers. + + The allele-grain sibling of ``GET /variants/{urn}``. Flat anchor identity (digest, level, HGVS, + ClinGen id, spec-pure VRS) plus the cross-layer equivalence class (each member labelled relative to + the focus) and a digest-keyed annotation map. The ``identifier`` may be: + + - a **VRS digest** (``ga4gh:VA.…``) — focuses that one allele. + - a **CAID** (``CA…``) — the nt-canonical change; The coding frame is the preferential focus, + falling back to the genomic frame if no coding frame exists. + - a **PAID** (``PA…``) — the protein change; the protein allele is focused and its nucleotide + equivalents surface as reverse-translation candidates. + + This is a **public molecular resource**. It carries no score-set-level information. No scores, + classifications, measurements, or version standing. Only the allele's own identity, its cross-layer + equivalence class, and public reference annotations (VEP / gnomAD / ClinVar). + """ + save_to_logging_context({"requested_resource": identifier, "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" + + if identifier.startswith("ga4gh:"): + allele = db.scalar(select(Allele).where(Allele.vrs_digest == identifier)) + if allele is None: + raise HTTPException(status_code=404, detail=f"allele with VRS digest '{identifier}' not found") + return get_allele_detail(db, allele, focus_digests={identifier}, as_of=as_of) + + # A ClinGen allele id (CAID or PAID): resolve to its allele(s) — the nt-canonical change's genomic + + # coding frames for a CAID, or the single protein allele for a PAID — and focus all of them. + matches = list(db.scalars(select(Allele).where(Allele.clingen_allele_id == identifier)).all()) + if not matches: + raise HTTPException(status_code=404, detail=f"no allele with ClinGen allele id '{identifier}' found") + + focus_digests = {a.vrs_digest for a in matches if a.vrs_digest is not None} + return get_allele_detail(db, _representative(matches), focus_digests=focus_digests, as_of=as_of) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 12fa5bf35..d3e8c4221 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -39,6 +39,7 @@ from mavedb.models import * # noqa: F403 from mavedb.routers import ( access_keys, + alleles, alphafold, api_information, clingen_alleles, @@ -92,6 +93,7 @@ ) app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) app.include_router(access_keys.router) +app.include_router(alleles.router) app.include_router(api_information.router) app.include_router(clingen_alleles.router) app.include_router(collections.router) @@ -240,6 +242,7 @@ def customize_openapi_schema(): openapi_schema["tags"] = [ access_keys.metadata, + alleles.metadata, api_information.metadata, clingen_alleles.metadata, collections.metadata, diff --git a/src/mavedb/view_models/allele_annotation.py b/src/mavedb/view_models/allele_annotation.py index b538d4639..a995fabcb 100644 --- a/src/mavedb/view_models/allele_annotation.py +++ b/src/mavedb/view_models/allele_annotation.py @@ -2,9 +2,9 @@ The pydantic serialization boundary over the ``lib.allele_annotations`` transit dataclasses (``from_attributes`` coerces them directly). These are the sparse, digest-keyed annotation blocks -that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and are the allele's own block -on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather than in either -one's file. +that ride *alongside* the spec-pure Cat-VRS on ``GET /variants/{urn}`` and alongside the cross-layer +equivalence class on ``GET /alleles/{digest}`` — so they live here, shared by both serving views rather +than in either one's file. """ from typing import Optional diff --git a/src/mavedb/view_models/allele_detail.py b/src/mavedb/view_models/allele_detail.py new file mode 100644 index 000000000..67ec74bd3 --- /dev/null +++ b/src/mavedb/view_models/allele_detail.py @@ -0,0 +1,39 @@ +"""Response view models for the allele-detail view (``GET /alleles/{digest|CAID}``). + +The pydantic serialization boundary over the :mod:`lib.allele_detail` transit dataclasses. The +allele-grain sibling of ``view_models.variant_detail``. The ``alleles`` map reuses the shared +:class:`AlleleIdentity` (``view_models.allele_identity``) and the digest-keyed +:class:`AlleleAnnotations` block (``view_models.allele_annotation``) — both shared with the +variant view. +""" + +from typing import Any, Optional + +from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity +from mavedb.view_models.base.base import BaseModel + + +class AlleleDetail(BaseModel): + """The allele-detail envelope (``GET /alleles/{digest|CAID}``). + + Flat anchor-identity fields (``digest`` / ``level`` / ``hgvs`` / ``clingenAlleleId`` and the spec-pure + GA4GH ``vrs`` variation) plus the MaveDB layer riding alongside. This layer, keyed by VRS digest, + contains the ``alleles`` map: the full cross-layer equivalence class. Each entry is an ``AlleleIdentity``, + labelled relative to the focus (``isFocus`` marks the queried allele / the CAID's representations) and + the digest-keyed ``annotations`` map. The two maps share keys. Measurement-agnostic: no score, + classification, or version standing, and no re-anchored Cat-VRS (those belong to ``GET /variants/{urn}``). + Absent fields are omitted. + """ + + digest: str + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + vrs: Optional[dict[str, Any]] = None + + alleles: dict[str, AlleleIdentity] = {} + annotations: dict[str, AlleleAnnotations] = {} + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/allele_identity.py b/src/mavedb/view_models/allele_identity.py new file mode 100644 index 000000000..b77743df5 --- /dev/null +++ b/src/mavedb/view_models/allele_identity.py @@ -0,0 +1,29 @@ +"""Response view model for the shared allele molecular-identity shape. + +The pydantic boundary over :class:`lib.allele_identity.AlleleIdentity` (``from_attributes`` coerces it; +aliases camelize per the shared base config). Shared by both serving views — the ``alleles`` map on +``GET /variants/{urn}`` and on ``GET /alleles/{digest}``. See the lib docstring for the focus-relative +semantics of each axis. +""" + +from typing import Optional + +from mavedb.view_models.base.base import BaseModel + + +class AlleleIdentity(BaseModel): + """One allele in a view's ``alleles`` map, keyed by VRS digest and labelled relative to the view's + focus allele. ``isFocus`` marks the anchor (measured allele / queried allele); ``relation`` and + ``derivation`` describe every other member's structural + provenance relationship to it, and are + absent on the focus itself.""" + + level: Optional[str] = None + hgvs: Optional[str] = None + clingen_allele_id: Optional[str] = None + is_focus: bool + relation: Optional[str] = None + derivation: Optional[str] = None + projection_of: Optional[str] = None + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index 31f5518aa..d9bc08344 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -10,6 +10,7 @@ from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.view_models.allele_annotation import AlleleAnnotations +from mavedb.view_models.allele_identity import AlleleIdentity from mavedb.view_models.base.base import BaseModel from mavedb.view_models.score_calibration import SavedFunctionalClassification @@ -30,34 +31,6 @@ class Config: from_attributes = True -class AlleleIdentity(BaseModel): - """The MaveDB molecular-identity facts for one of the variant's linked alleles. - - An entry of the ``alleles`` sidecar, keyed by VRS digest. ``level`` + ``hgvs`` (the reference-frame - HGVS) are what the UI labels the per-level annotation panel by — never the digest. - - Three independent axes: - - ``relation`` (Cat-VRS, structural): member→defining relation; ``null`` when it *is* the measured - allele, or when the allele is not a Cat-VRS member. - - ``derivation`` (provenance): ``authoritative`` (measured) / ``projection`` (deterministic, - precise, derived from the measured change) / ``candidate`` (protein-assay reverse-translation, - ambiguous) / ``convergent`` (a distinct, precise nucleotide change that converges on the measured - protein consequence). Orthogonal to ``relation``. Do not conflate them. - - ``projectionOf`` (provenance): the VRS digest of this allele's projection sibling (the paired c↔g member of - its projection pair group); ``null`` for the protein apex and pre-reverse-translation data. - """ - - level: Optional[str] = None - hgvs: Optional[str] = None - clingen_allele_id: Optional[str] = None - relation: Optional[str] = None - derivation: Optional[str] = None - projection_of: Optional[str] = None - - class Config: - from_attributes = True - - class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``). diff --git a/tests/lib/test_allele_detail.py b/tests/lib/test_allele_detail.py new file mode 100644 index 000000000..75375b7a4 --- /dev/null +++ b/tests/lib/test_allele_detail.py @@ -0,0 +1,203 @@ +# ruff: noqa: E402 +"""Integration tests for the allele-detail assembly backing ``GET /alleles/{digest}`` (+ CAID). + +``get_allele_detail`` is the allele-grain sibling of ``get_variant_detail``: it anchors on an allele +(a digest, or a CAID's genomic+coding pair) and serves the **full cross-layer equivalence class** +(``get_allele_translations``), with each member labelled *relative to the focus* — faithful +(``projection``) vs. convergent cousin vs. reverse-translation ``candidate`` — plus the digest-keyed +annotations. These tests pin the labelling in both directions (nt focus / protein focus), the full-class +membership incl. cousins, the CAID focus set, the orphan fallback, and ``as_of``. +""" + +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.lib.allele_detail import get_allele_detail +from mavedb.models.allele import Allele +from mavedb.models.variant import Variant +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + +T0 = datetime(2020, 1, 1, tzinfo=timezone.utc) +T1 = datetime(2021, 1, 1, tzinfo=timezone.utc) +T2 = datetime(2022, 1, 1, tzinfo=timezone.utc) + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _variant(session, score_set, suffix): + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#{suffix}", score_set_id=score_set.id) + session.add(variant) + session.commit() + return variant + + +def _allele(session, digest) -> Allele: + allele = session.scalar(select(Allele).where(Allele.vrs_digest == digest)) + assert allele is not None, f"allele {digest!r} not seeded" + return allele + + +def _coding_measured_specs(): + """A coding-measured record: authoritative cdna (+VEP+ClinGen), its genomic projection partner + (shared projection_group 0), the protein apex, and a synonymous cousin (own projection_group 1).""" + return [ + AlleleSpec( + digest="cdna-d", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest="gen-d", + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec(digest="prot-d", level="protein", hgvs_p="NP_000537.3:p.Ala406Thr", post_mapped=_post_mapped()), + AlleleSpec( + digest="cousin-d", + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ] + + +@pytest.mark.integration +def test_nt_focus_labels_the_full_class_relative_to_the_queried_allele(session, setup_lib_db_with_score_set): + """Fetching the measured coding allele: the whole class comes back, each member labelled relative to + it — its genomic is a faithful coordinate projection, the protein its consequence, the cousin convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert detail.digest == "cdna-d" + assert detail.level == "cdna" + assert detail.vrs is not None and detail.vrs["type"] == "Allele" + assert set(detail.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} # full class, cousin included + + focus = detail.alleles["cdna-d"] + assert focus.is_focus is True and focus.relation is None and focus.derivation is None + assert focus.projection_of == "gen-d" + + genomic = detail.alleles["gen-d"] + assert genomic.is_focus is False + assert genomic.relation == "coordinate_representation_of" + assert genomic.derivation == "projection" + assert genomic.projection_of == "cdna-d" + + protein = detail.alleles["prot-d"] + assert protein.relation == "translation_of" + assert protein.derivation == "projection" + + cousin = detail.alleles["cousin-d"] + assert cousin.relation == "co_encodes" + assert cousin.derivation == "convergent" # distinct change sharing the consequence, not the coordinate partner + + +@pytest.mark.integration +def test_protein_focus_labels_every_nucleotide_as_a_candidate(session, setup_lib_db_with_score_set): + """Fetching the protein apex: walking down is ambiguous, so every nt member is a reverse-translation + candidate (the 'less power without a measurement' case, still labelled precisely by direction).""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "prot-d"), focus_digests={"prot-d"}) + + assert detail.alleles["prot-d"].is_focus is True + for nt in ("cdna-d", "gen-d", "cousin-d"): + assert detail.alleles[nt].relation == "encodes", nt + assert detail.alleles[nt].derivation == "candidate", nt + + +@pytest.mark.integration +def test_annotations_join_by_digest(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d"}) + + assert set(detail.annotations) == set(detail.alleles) + assert detail.annotations["cdna-d"].vep is not None + assert detail.annotations["cdna-d"].vep.consequence == "missense_variant" + assert detail.annotations["gen-d"].vep is None # a member with no annotations still gets an empty block + + +@pytest.mark.integration +def test_caid_focus_flags_both_frames(session, setup_lib_db_with_score_set): + """A CAID names the nt change in both frames: fetched by CAID, the genomic + coding are both isFocus + (each other's coordinate partner), and only the true cousin is convergent.""" + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna") + + # The CAID resolves to the {cdna, genomic} pair (both carry clingen_allele_id="CA1"). + detail = get_allele_detail(session, _allele(session, "cdna-d"), focus_digests={"cdna-d", "gen-d"}) + + assert detail.alleles["cdna-d"].is_focus is True + assert detail.alleles["gen-d"].is_focus is True + assert detail.alleles["prot-d"].relation == "translation_of" + assert detail.alleles["cousin-d"].derivation == "convergent" + + +@pytest.mark.integration +def test_orphan_allele_falls_back_to_itself(session, setup_lib_db_with_score_set): + orphan = Allele(vrs_digest="orphan-d", level="cdna", hgvs_c="NM:c.9A>G", post_mapped=_post_mapped()) + session.add(orphan) + session.commit() + + detail = get_allele_detail(session, orphan, focus_digests={"orphan-d"}) + + assert set(detail.alleles) == {"orphan-d"} + assert detail.alleles["orphan-d"].is_focus is True + assert set(detail.annotations) == {"orphan-d"} + + +@pytest.mark.integration +def test_as_of_reconstructs_membership(session, setup_lib_db_with_score_set): + variant = _variant(session, setup_lib_db_with_score_set, 1) + seed_mapping_record(session, variant, alleles=_coding_measured_specs(), assay_level="cdna", valid_from=T1) + + anchor = _allele(session, "cdna-d") + + before = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T0) + assert set(before.alleles) == {"cdna-d"} # links not yet live → class collapses to the focus + assert before.annotations["cdna-d"].vep is None + + after = get_allele_detail(session, anchor, focus_digests={"cdna-d"}, as_of=T2) + assert set(after.alleles) == {"cdna-d", "gen-d", "prot-d", "cousin-d"} + assert after.annotations["cdna-d"].vep is not None diff --git a/tests/lib/test_variant_detail.py b/tests/lib/test_variant_detail.py index 4c8a3384c..8950dd6ef 100644 --- a/tests/lib/test_variant_detail.py +++ b/tests/lib/test_variant_detail.py @@ -185,10 +185,13 @@ def test_full_envelope_for_a_coding_assay(session, setup_lib_db_with_score_set): assert measured_identity.hgvs == "NM_000546.6:c.1216G>A" # coalesced from hgvs_c assert measured_identity.clingen_allele_id == "CA123" assert measured_identity.relation is None # the defining allele has no relation to itself - # The measured allele is authoritative and pairs with its genomic projection sibling. - assert measured_identity.derivation == "authoritative" + # The measured allele is the focus; derivation describes the *other* members relative to it, so the + # focus carries none. It pairs with its genomic projection sibling. + assert measured_identity.is_focus is True + assert measured_identity.derivation is None assert measured_identity.projection_of == "gen-digest" genomic_identity = detail.alleles["gen-digest"] + assert genomic_identity.is_focus is False # Nucleotide assay: the genomic sibling is a precise projection, paired back to the measured cdna. assert genomic_identity.level == "genomic" assert genomic_identity.derivation == "projection" @@ -245,11 +248,12 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor assert detail.reference_hgvs == "NP_000537.3:p.Ala406Thr" assert detail.assay_level_digest == "prot-digest" assert detail.mode == "reverse_translation" - # Defining protein allele has no relation to itself, is authoritative, and is in no c/g pair. + # Defining protein allele is the focus (no relation/derivation to itself), and is in no c/g pair. prot = detail.alleles["prot-digest"] assert prot.level == "protein" assert prot.relation is None - assert prot.derivation == "authoritative" + assert prot.is_focus is True + assert prot.derivation is None assert prot.projection_of is None # The coding member `encodes` the protein (structural relation) but is an ambiguous `candidate` # (provenance) — the two axes disagree by design — and pairs to its genomic projection. @@ -265,9 +269,9 @@ def test_protein_assay_targets_the_protein_frame(session, setup_lib_db_with_scor @pytest.mark.integration def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, setup_lib_db_with_score_set): - """Links written before reverse translation ran carry a NULL projection_group: derivation is still - computed (authoritative / projection from is_authoritative + assay level), but projection_of degrades - to null — nothing pairs until the data is re-processed.""" + """Links written before reverse translation ran carry a NULL projection_group: focus/derivation are + still computed (is_focus from is_authoritative; projection from the assay level), but projection_of + degrades to null — nothing pairs until the data is re-processed.""" score_set = setup_lib_db_with_score_set variant = _variant(session, score_set, 1, hgvs_nt="c.1216G>A") record = _record(session, variant, assay_level="cdna", hgvs_assay_level="NM_000546.6:c.1216G>A") @@ -278,7 +282,8 @@ def test_pre_reverse_translation_data_degrades_projection_of_to_null(session, se detail = get_variant_detail(session, variant) - assert detail.alleles["cdna-digest"].derivation == "authoritative" + assert detail.alleles["cdna-digest"].is_focus is True + assert detail.alleles["cdna-digest"].derivation is None assert detail.alleles["cdna-digest"].projection_of is None # no group -> no sibling assert detail.alleles["prot-digest"].derivation == "projection" assert detail.alleles["prot-digest"].projection_of is None diff --git a/tests/routers/test_allele.py b/tests/routers/test_allele.py new file mode 100644 index 000000000..7ce80c0c9 --- /dev/null +++ b/tests/routers/test_allele.py @@ -0,0 +1,224 @@ +# ruff: noqa: E402 +"""Router tests for the allele-detail endpoints (``GET /alleles/{digest}`` and ``?clingenAlleleId=``). + +Exercise the HTTP surface end to end: the envelope serializes and validates, the equivalence class is +labelled relative to the focus, CAID fetch focuses the nt-canonical change, ``as_of`` is echoed and +reconstructs membership, an unknown id is a plain 404 — and, unlike ``GET /variants/{urn}``, the +endpoint is **public with no privacy gate**: an allele reachable only through a private score set is +still served. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from mavedb.view_models.allele_detail import AlleleDetail +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import create_seq_score_set_with_variants +from tests.helpers.util.user import change_ownership + +_VALID_DIGEST = "0123456789abcdefghijABCDEFGHIJ_-" + + +def _ir(tag: str) -> str: + """A valid GA4GH IR (``ga4gh:VA.`` + exactly 32 base64url chars), tagged for readability.""" + return f"ga4gh:VA.{tag.ljust(32, '0')}" + + +CDNA = _ir("cdna") +GEN = _ir("gen") +PROT = _ir("prot") +COUSIN = _ir("cousin") + + +def _post_mapped() -> dict: + return { + "id": f"ga4gh:VA.{_VALID_DIGEST}", + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": _VALID_DIGEST, + "location": { + "id": f"ga4gh:SL.{_VALID_DIGEST}", + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": _VALID_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NP_000000.0", + "refgetAccession": "SQ.0123456789abcdefghijABCDEFGHIJ_-", + }, + }, + } + + +def _seed(session, variant_urn): + """Coding-measured record: authoritative cdna (+ClinGen CA1 +VEP), its genomic partner (also CA1), + the protein apex, and a synonymous cousin in its own projection group.""" + seed_mapping_record( + session, + variant_urn, + alleles=[ + AlleleSpec( + digest=CDNA, + level="cdna", + is_authoritative=True, + clingen_allele_id="CA1", + hgvs_c="NM_000546.6:c.1216G>A", + post_mapped=_post_mapped(), + vep_consequence="missense_variant", + projection_group=0, + ), + AlleleSpec( + digest=GEN, + level="genomic", + clingen_allele_id="CA1", + hgvs_g="NC_000017.11:g.7676154C>T", + post_mapped=_post_mapped(), + projection_group=0, + ), + AlleleSpec( + digest=PROT, + level="protein", + clingen_allele_id="PA1", + hgvs_p="NP_000537.3:p.Ala406Thr", + post_mapped=_post_mapped(), + ), + AlleleSpec( + digest=COUSIN, + level="cdna", + hgvs_c="NM_000546.6:c.1218C>T", + post_mapped=_post_mapped(), + projection_group=1, + ), + ], + assay_level="cdna", + ) + + +def test_get_allele_detail_envelope(client, session, data_provider, data_files, setup_router_db): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + + assert body["digest"] == CDNA + assert body["level"] == "cdna" + assert body["hgvs"] == "NM_000546.6:c.1216G>A" + assert body["clingenAlleleId"] == "CA1" + assert body["vrs"]["type"] == "Allele" + # Full equivalence class, keyed by digest, labelled relative to the queried allele (camelCase). + assert set(body["alleles"]) == {CDNA, GEN, PROT, COUSIN} + assert body["alleles"][CDNA]["isFocus"] is True + assert "derivation" not in body["alleles"][CDNA] # focus carries none; dropped by exclude_none + assert body["alleles"][GEN]["relation"] == "coordinate_representation_of" + assert body["alleles"][GEN]["derivation"] == "projection" + assert body["alleles"][PROT]["relation"] == "translation_of" + assert body["alleles"][COUSIN]["relation"] == "co_encodes" + assert body["alleles"][COUSIN]["derivation"] == "convergent" + # Annotations share keys with the class and join by digest. + assert set(body["annotations"]) == {CDNA, GEN, PROT, COUSIN} + assert body["annotations"][CDNA]["vep"]["consequence"] == "missense_variant" + # Measurement-agnostic: no variant-detail measurement fields. + assert "molecularRepresentation" not in body + assert "classifications" not in body + assert "isCurrent" not in body + assert response.headers["X-As-Of"] == "current" + + +def test_get_allele_detail_by_caid(client, session, data_provider, data_files, setup_router_db): + """CAID fetch on the overloaded `/alleles/{identifier}` path focuses the nt-canonical change (both + genomic + coding frames isFocus).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/CA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "CA1" + assert body["alleles"][CDNA]["isFocus"] is True + assert body["alleles"][GEN]["isFocus"] is True # the CAID's other frame + assert body["alleles"][COUSIN]["derivation"] == "convergent" + + +def test_get_allele_detail_by_paid(client, session, data_provider, data_files, setup_router_db): + """PAID fetch focuses the protein allele; its nucleotide equivalents surface as candidates — the + same `/alleles/{identifier}` path, no separate route or param.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get("/api/v1/alleles/PA1") + + assert response.status_code == 200 + body = response.json() + AlleleDetail.model_validate(body) + assert body["clingenAlleleId"] == "PA1" + assert body["alleles"][PROT]["isFocus"] is True + # Walking down from the protein is ambiguous: every nucleotide member is a candidate. + for nt in (CDNA, GEN, COUSIN): + assert body["alleles"][nt]["derivation"] == "candidate", nt + + +def test_get_allele_detail_unknown_digest_is_404(client, setup_router_db): + response = client.get(f"/api/v1/alleles/{_ir('nonexistent')}") + assert response.status_code == 404 + + +def test_get_allele_detail_unknown_caid_is_404(client, setup_router_db): + response = client.get("/api/v1/alleles/CA999999") + assert response.status_code == 404 + + +def test_get_allele_detail_is_public_no_privacy_gate( + client, session, data_provider, data_files, setup_router_db, anonymous_app_overrides +): + """The decision: no privacy gate. An allele reachable only through a private score set is still + served to an anonymous caller — contrast ``GET /variants/{urn}``, which 404s the same variant.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + change_ownership(session, score_set["urn"], ScoreSetDbModel) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/alleles/{CDNA}") + + assert response.status_code == 200 + assert response.json()["digest"] == CDNA + + +def test_get_allele_detail_echoes_as_of_header(client, session, data_provider, data_files, setup_router_db): + """``as_of`` is echoed and reconstructs membership: before the record was live the class collapses + to the focus (the allele row itself is content-addressed, so it still resolves).""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + _seed(session, f"{score_set['urn']}#1") + + response = client.get(f"/api/v1/alleles/{CDNA}", params={"as_of": "2020-01-01T00:00:00Z"}) + + assert response.status_code == 200 + assert response.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" + assert set(response.json()["alleles"]) == {CDNA} diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index dae7fd50e..a422bbe24 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -141,9 +141,13 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none - # Provenance axis (derivation) + the projection pairing (projectionOf), camelCased. - assert body["alleles"]["cdna-digest"]["derivation"] == "authoritative" + # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the + # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). + assert body["alleles"]["cdna-digest"]["isFocus"] is True + assert "derivation" not in body["alleles"]["cdna-digest"] # null derivation dropped by exclude_none assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" + assert body["alleles"]["gen-digest"]["isFocus"] is False + assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" assert body["alleles"]["gen-digest"]["derivation"] == "projection" assert body["alleles"]["gen-digest"]["projectionOf"] == "cdna-digest" assert body["alleles"]["prot-digest"]["level"] == "protein" From 193c55f4dc99af580a7389328c32df2a65d6920a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 10:27:08 -0700 Subject: [PATCH 81/93] refactor(workflow): extract shared pipeline-kickoff helper from score_sets router score_sets.py had its own private `enqueue_pipeline_entrypoint` helper for building `Pipeline`/`JobRun` records and enqueuing the `start_pipeline` ARQ job, duplicated at both of its call sites. Move that logic into `mavedb.lib.workflow.kickoff.enqueue_pipeline_for_score_set` so the `run_pipeline` script and other future callers share the same enqueue contract instead of reimplementing it. - Restore the discard-pipeline-on-enqueue-failure safety net that the router's original helper had but the extracted version initially dropped, so a failed ARQ enqueue never leaves an orphaned pipeline behind in the database. - Add unit and integration tests for `enqueue_pipeline_for_score_set`, covering correlation-id precedence, param merging, ARQ dedup handling, and pipeline cleanup on enqueue failure. --- src/mavedb/lib/workflow/kickoff.py | 96 +++++++++++ src/mavedb/routers/score_sets.py | 61 ++----- tests/lib/workflow/test_kickoff.py | 258 +++++++++++++++++++++++++++++ tests/routers/test_score_set.py | 52 ++++-- 4 files changed, 406 insertions(+), 61 deletions(-) create mode 100644 src/mavedb/lib/workflow/kickoff.py create mode 100644 tests/lib/workflow/test_kickoff.py diff --git a/src/mavedb/lib/workflow/kickoff.py b/src/mavedb/lib/workflow/kickoff.py new file mode 100644 index 000000000..94feec0bf --- /dev/null +++ b/src/mavedb/lib/workflow/kickoff.py @@ -0,0 +1,96 @@ +"""Create and enqueue a named pipeline for a single score set. + +The one shared kickoff seam: build the ``Pipeline`` / ``JobRun`` / ``JobDependency`` records via +:class:`PipelineFactory`, then enqueue the ``start_pipeline`` entrypoint in ARQ. Both the ad-hoc +``run_pipeline`` script and the batch ``enrich_backfilled_score_sets`` driver go through here so the +enqueue contract (entrypoint id + :func:`arq_job_id`) lives in exactly one place. +""" + +from __future__ import annotations + +import datetime +import logging +from typing import Any, Optional + +from arq.connections import ArqRedis +from arq.jobs import Job +from sqlalchemy.orm import Session + +from mavedb.lib.logging.context import correlation_id_for_context, logging_context, save_to_logging_context +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.worker.lib.managers.utils import arq_job_id + +logger = logging.getLogger(__name__) + + +async def enqueue_pipeline_for_score_set( + db: Session, + redis: ArqRedis, + *, + pipeline_name: str, + score_set: ScoreSet, + user: User, + extra_params: Optional[dict[str, Any]] = None, +) -> tuple[Pipeline, Optional[Job]]: + """Create the pipeline records for ``score_set`` and enqueue its ``start_pipeline`` entrypoint. + + Returns the created :class:`Pipeline` and the enqueued ARQ :class:`Job` (``None`` when ARQ + deduplicated an already-queued job with the same id). Raises ``KeyError`` for an unknown + ``pipeline_name`` and ``ValueError`` for an invalid pipeline definition, surfaced from + :meth:`PipelineFactory.create_pipeline`. If the ARQ enqueue itself raises, the just-created + pipeline records are discarded (via :meth:`PipelineFactory.discard_pipeline`) before the + exception is re-raised, so a failed enqueue never leaves an orphaned pipeline behind. + """ + correlation_id = ( + correlation_id_for_context() + or f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + ) + params: dict[str, Any] = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + if extra_params: + params.update(extra_params) + + factory = PipelineFactory(session=db) + pipeline, entrypoint = factory.create_pipeline( + pipeline_name=pipeline_name, creating_user=user, pipeline_params=params + ) + job_id = arq_job_id(entrypoint) + + try: + job = await redis.enqueue_job(entrypoint.job_function, entrypoint.id, _job_id=job_id) + except Exception: + logger.error( + "Failed to enqueue pipeline %s for score set %s; discarding pipeline %s", + pipeline_name, + score_set.urn, + pipeline.id, + extra=logging_context(), + ) + factory.discard_pipeline(pipeline) + raise + + save_to_logging_context({"pipeline_id": pipeline.id, "job_id": job_id}) + + if not job: + logger.info( + "Pipeline %s for score set %s already enqueued (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + else: + logger.info( + "Enqueued pipeline %s for score set %s (job id: %s)", + pipeline_name, + score_set.urn, + job_id, + extra=logging_context(), + ) + return pipeline, job diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 506ed0b5b..7baeb0893 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -44,11 +44,7 @@ find_or_create_publication_identifier, ) from mavedb.lib.logging import LoggedRoute -from mavedb.lib.logging.context import ( - correlation_id_for_context, - logging_context, - save_to_logging_context, -) +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_set_variants import get_lean_score_set_variants @@ -75,14 +71,13 @@ generate_experiment_urn, generate_score_set_urn, ) -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.pipeline import Pipeline from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -105,7 +100,6 @@ from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate -from mavedb.worker.lib.managers.utils import arq_job_id TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -114,37 +108,6 @@ SCORE_SET_SEARCH_MAX_PUBLICATION_IDENTIFIERS = 40 -async def enqueue_pipeline_entrypoint( - *, - db: Session, - worker: ArqRedis, - pipeline_name: str, - creating_user: Any, - pipeline_params: dict[str, Any], -) -> None: - pipeline_factory = PipelineFactory(session=db) - pipeline: Optional[Pipeline] = None - - try: - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=creating_user, - pipeline_params=pipeline_params, - ) - job = await worker.enqueue_job( - pipeline_entrypoint.job_function, pipeline_entrypoint.id, _job_id=arq_job_id(pipeline_entrypoint) - ) - except Exception: - pipeline_factory.discard_pipeline(pipeline) - raise - - if job is None: - logger.info(msg=f"Pipeline entrypoint for {pipeline_name} has already been enqueued.", extra=logging_context()) - else: - save_to_logging_context({"worker_job_id": job.job_id}) - logger.info(msg=f"Enqueued pipeline entrypoint for {pipeline_name}.", extra=logging_context()) - - async def enqueue_variant_creation( *, item: ScoreSet, @@ -212,15 +175,13 @@ async def enqueue_variant_creation( ) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="validate_map_annotate_score_set", - creating_user=user_data.user, - pipeline_params={ - "correlation_id": correlation_id_for_context(), - "score_set_id": item.id, - "updater_id": user_data.user.id, + score_set=item, + user=user_data.user, + extra_params={ "scores_file_key": scores_file_key, "counts_file_key": counts_file_key, "score_columns_metadata": item.dataset_columns.get("score_columns_metadata") @@ -2461,12 +2422,12 @@ async def publish_score_set( db.refresh(item) try: - await enqueue_pipeline_entrypoint( + await enqueue_pipeline_for_score_set( db=db, - worker=worker, + redis=worker, pipeline_name="publish_score_set", - creating_user=user_data.user, - pipeline_params={"correlation_id": correlation_id_for_context(), "score_set_id": item.id}, + score_set=item, + user=user_data.user, ) except Exception as exc: logger.warning( diff --git a/tests/lib/workflow/test_kickoff.py b/tests/lib/workflow/test_kickoff.py new file mode 100644 index 000000000..9b98eaf45 --- /dev/null +++ b/tests/lib/workflow/test_kickoff.py @@ -0,0 +1,258 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("arq") + +from unittest.mock import AsyncMock, Mock, patch + +from arq.jobs import Job as ArqJob +from sqlalchemy import select + +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.user import User +from tests.helpers.constants import TEST_USER + + +def _mock_entrypoint(*, id=42, job_function="start_pipeline", urn="urn:mavedb:job-fixed", retry_count=0): + return Mock(id=id, job_function=job_function, urn=urn, retry_count=retry_count) + + +@pytest.mark.unit +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetUnit: + """Unit tests with PipelineFactory and ARQ mocked out.""" + + @pytest.fixture(autouse=True) + def setup(self): + self.db = Mock() + self.redis = AsyncMock() + self.score_set = Mock(id=99, urn="urn:mavedb:00000001-a-1") + self.user = Mock(id=5) + self.pipeline = Mock(spec=Pipeline) + self.entrypoint = _mock_entrypoint() + + factory_patch = patch("mavedb.lib.workflow.kickoff.PipelineFactory") + self.factory_cls = factory_patch.start() + self.factory = self.factory_cls.return_value + self.factory.create_pipeline.return_value = (self.pipeline, self.entrypoint) + yield + factory_patch.stop() + + async def _call(self, **kwargs): + return await enqueue_pipeline_for_score_set( + self.db, self.redis, pipeline_name="test_pipeline", score_set=self.score_set, user=self.user, **kwargs + ) + + async def test_returns_pipeline_and_job_when_enqueued(self): + job = Mock(spec=ArqJob) + self.redis.enqueue_job.return_value = job + + result = await self._call() + + assert result == (self.pipeline, job) + self.redis.enqueue_job.assert_awaited_once_with( + self.entrypoint.job_function, + self.entrypoint.id, + _job_id=f"{self.entrypoint.urn}#{self.entrypoint.retry_count}", + ) + + async def test_returns_none_job_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + result = await self._call() + + assert result == (self.pipeline, None) + + async def test_propagates_key_error_for_unknown_pipeline_name(self): + self.factory.create_pipeline.side_effect = KeyError("unknown_pipeline") + + with pytest.raises(KeyError): + await self._call() + + async def test_discards_pipeline_and_reraises_when_enqueue_raises(self): + self.redis.enqueue_job.side_effect = ConnectionError("redis is unreachable") + + with pytest.raises(ConnectionError): + await self._call() + + self.factory.discard_pipeline.assert_called_once_with(self.pipeline) + + async def test_does_not_discard_pipeline_when_enqueue_succeeds(self): + self.redis.enqueue_job.return_value = Mock(spec=ArqJob) + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_does_not_discard_pipeline_when_arq_deduplicates(self): + self.redis.enqueue_job.return_value = None + + await self._call() + + self.factory.discard_pipeline.assert_not_called() + + async def test_default_correlation_id_built_from_pipeline_name_score_set_and_user(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value=None): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"].startswith(f"test_pipeline_{self.score_set.urn}_{self.user.id}_") + + async def test_correlation_id_from_context_used_when_present(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call() + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "context-correlation-id" + + async def test_extra_params_correlation_id_overrides_context(self): + with patch("mavedb.lib.workflow.kickoff.correlation_id_for_context", return_value="context-correlation-id"): + await self._call(extra_params={"correlation_id": "explicit-correlation-id"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["correlation_id"] == "explicit-correlation-id" + + async def test_extra_params_merged_alongside_base_params(self): + await self._call(extra_params={"some_extra_param": "some_value"}) + + params = self.factory.create_pipeline.call_args.kwargs["pipeline_params"] + assert params["some_extra_param"] == "some_value" + assert params["score_set_id"] == self.score_set.id + assert params["updater_id"] == self.user.id + + async def test_create_pipeline_called_with_pipeline_name_and_user(self): + await self._call() + + call_kwargs = self.factory.create_pipeline.call_args.kwargs + assert call_kwargs["pipeline_name"] == "test_pipeline" + assert call_kwargs["creating_user"] is self.user + + +@pytest.mark.integration +@pytest.mark.asyncio +class TestEnqueuePipelineForScoreSetIntegration: + """Integration tests exercising the real PipelineFactory and a fakeredis-backed ARQ instance.""" + + @pytest.fixture(autouse=True) + def setup(self, session, setup_lib_db_with_score_set): + self.score_set = setup_lib_db_with_score_set + self.user = session.query(User).filter(User.username == TEST_USER["username"]).first() + + async def test_creates_pipeline_and_job_run_records_and_enqueues_job( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, job = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert job is not None + assert session.execute(select(Pipeline).where(Pipeline.id == pipeline.id)).scalar_one() is pipeline + + job_run = session.execute( + select(JobRun).where(JobRun.pipeline_id == pipeline.id, JobRun.job_function == "process_data") + ).scalar_one() + assert job_run.job_params["required_param"] == "some_value" + + async def test_correlation_id_defaults_to_generated_value_without_request_context( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline, _ = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline.correlation_id.startswith( + f"{sample_independent_pipeline_definition['name']}_{self.score_set.urn}_{self.user.id}_" + ) + + async def test_independent_calls_create_independent_pipelines_and_jobs( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + assert pipeline_one.id != pipeline_two.id + assert job_one is not None + assert job_two is not None + + async def test_returns_none_job_when_arq_job_id_collides( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with patch("mavedb.lib.workflow.kickoff.arq_job_id", return_value="fixed-dedupe-id"): + pipeline_one, job_one = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + pipeline_two, job_two = await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # ARQ deduplicates the second enqueue on the collided job id, but pipeline records are + # still created for both calls -- kickoff does not roll back on dedup. + assert job_one is not None + assert job_two is None + assert pipeline_one.id != pipeline_two.id + + async def test_unknown_pipeline_name_raises_key_error(self, session, arq_redis): + with pytest.raises(KeyError): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name="does_not_exist_pipeline", + score_set=self.score_set, + user=self.user, + ) + + async def test_discards_pipeline_records_when_enqueue_raises( + self, session, arq_redis, with_test_pipeline_definition_ctx, sample_independent_pipeline_definition + ): + with ( + patch.object(arq_redis, "enqueue_job", AsyncMock(side_effect=ConnectionError("redis is unreachable"))), + pytest.raises(ConnectionError), + ): + await enqueue_pipeline_for_score_set( + session, + arq_redis, + pipeline_name=sample_independent_pipeline_definition["name"], + score_set=self.score_set, + user=self.user, + extra_params={"required_param": "some_value"}, + ) + + # The pipeline (and its job runs) created before the failed enqueue must not be left behind. + assert session.execute(select(Pipeline)).scalars().all() == [] + assert session.execute(select(JobRun)).scalars().all() == [] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 11a07bf51..77150ba4c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -19,12 +19,12 @@ from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.enums.target_category import TargetCategory from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline -from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from mavedb.view_models.lean_variant import LeanVariant @@ -64,6 +64,7 @@ VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.common import ( deepcamelize, parse_ndjson_response, @@ -76,7 +77,6 @@ create_publish_and_promote_score_calibration, create_test_score_calibration_in_score_set_via_client, ) -from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record from tests.helpers.util.score_set import ( create_seq_score_set, create_seq_score_set_with_mapped_variants, @@ -1537,7 +1537,7 @@ def test_upload_score_set_variant_data_deletes_s3_files_when_pipeline_creation_f with ( open(scores_csv_path, "rb") as scores_file, - patch("mavedb.routers.score_sets.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), + patch("mavedb.lib.workflow.kickoff.PipelineFactory.create_pipeline", side_effect=Exception("pipeline failure")), patch.object(mock_s3_client, "upload_fileobj", return_value=None), ): response = client.post( @@ -2004,7 +2004,10 @@ def test_multiple_score_set_meta_analysis_single_experiment( published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2044,7 +2047,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2086,7 +2092,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiments( ) published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2142,7 +2151,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco published_score_set_1_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1_1['urn']}")).json() assert meta_score_set_1["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1_1["urn"], published_score_set_1_2["urn"]] + [ + published_score_set_1_1["urn"], + published_score_set_1_2["urn"], + ] ) assert published_score_set_1_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_1["urn"]] @@ -2159,7 +2171,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_different_sco ) published_score_set_2_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_2_1['urn']}")).json() assert meta_score_set_2["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_2_1["urn"], published_score_set_2_2["urn"]] + [ + published_score_set_2_1["urn"], + published_score_set_2_2["urn"], + ] ) assert published_score_set_2_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set_2["urn"]] @@ -2281,7 +2296,10 @@ def test_multiple_score_set_meta_analysis_single_experiment_with_different_creat published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -2323,7 +2341,10 @@ def test_multiple_score_set_meta_analysis_multiple_experiment_sets_with_differen published_score_set_1_refresh = (client.get(f"/api/v1/score-sets/{published_score_set_1['urn']}")).json() assert meta_score_set["metaAnalyzesScoreSetUrns"] == sorted( - [published_score_set_1["urn"], published_score_set_2["urn"]] + [ + published_score_set_1["urn"], + published_score_set_2["urn"], + ] ) assert published_score_set_1_refresh["metaAnalyzedByScoreSetUrns"] == [meta_score_set["urn"]] @@ -3532,7 +3553,16 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r download_scores_and_counts_csv = download_scores_and_counts_csv_response.text reader = csv.DictReader(StringIO(download_scores_and_counts_csv)) assert sorted(reader.fieldnames) == sorted( - ["accession", "hgvs_nt", "hgvs_pro", "scores.score", "scores.s_0", "scores.s_1", "counts.c_0", "counts.c_1"] + [ + "accession", + "hgvs_nt", + "hgvs_pro", + "scores.score", + "scores.s_0", + "scores.s_1", + "counts.c_0", + "counts.c_1", + ] ) From a71226241ec87c7f2b0d3d2d0cbbfc9506178106 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 10:44:50 -0700 Subject: [PATCH 82/93] fixup ~2 --- src/mavedb/lib/allele_detail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mavedb/lib/allele_detail.py b/src/mavedb/lib/allele_detail.py index 14603f509..b81b0db91 100644 --- a/src/mavedb/lib/allele_detail.py +++ b/src/mavedb/lib/allele_detail.py @@ -1,4 +1,4 @@ -"""The allele-detail view backing ``GET /alleles/{digest:CAID}``. +"""The allele-detail view backing ``GET /alleles/{digest|CAID}``. The allele-grain sibling of :mod:`lib.variant_detail`. Where the variant view anchors on a *measurement*, this anchors on an *allele* (a VRS digest, or a CAID naming the nt-canonical change) From c87b72fa473d28445da018031e7b6a99344c5e9e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 11:48:30 -0700 Subject: [PATCH 83/93] feat(scripts): add batch driver for score-set mapping/enrichment pipelines - Add run_score_set_pipelines.py, which selects unmapped or unenriched score sets and enqueues the appropriate pipeline for each, ordered by gene to maximize ClinGen CAR cache reuse - Refactor run_pipeline.py to share pipeline creation/enqueueing logic via the new enqueue_pipeline_for_score_set helper instead of duplicating PipelineFactory/redis wiring inline --- src/mavedb/scripts/run_pipeline.py | 41 +-- src/mavedb/scripts/run_score_set_pipelines.py | 276 ++++++++++++++++++ 2 files changed, 288 insertions(+), 29 deletions(-) create mode 100644 src/mavedb/scripts/run_score_set_pipelines.py diff --git a/src/mavedb/scripts/run_pipeline.py b/src/mavedb/scripts/run_pipeline.py index c947fe122..658037fbb 100644 --- a/src/mavedb/scripts/run_pipeline.py +++ b/src/mavedb/scripts/run_pipeline.py @@ -11,7 +11,6 @@ poetry run python -m mavedb.scripts.run_pipeline --list """ -import datetime import logging import sys @@ -21,10 +20,9 @@ from mavedb.db.session import SessionLocal from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS -from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.score_set import ScoreSet from mavedb.models.user import User -from mavedb.worker.lib.managers.utils import arq_job_id from mavedb.worker.settings import RedisWorkerSettings logger = logging.getLogger(__name__) @@ -108,40 +106,25 @@ async def main( click.echo(f"User not found: {resolved_updater_id}", err=True) sys.exit(1) - correlation_id = f"{pipeline_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" - pipeline_params: dict = { - "correlation_id": correlation_id, - "score_set_id": score_set.id, - "updater_id": user.id, - } - for key, value in extra_params: - pipeline_params[key] = value - - try: - pipeline_factory = PipelineFactory(session=db) - pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( - pipeline_name=pipeline_name, - creating_user=user, - pipeline_params=pipeline_params, - ) - except (KeyError, ValueError) as e: - click.echo(f"Failed to create pipeline: {e}", err=True) - sys.exit(1) - - click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={correlation_id})") - # Connect to Redis and enqueue redis = await create_pool(RedisWorkerSettings) try: - job = await redis.enqueue_job( - pipeline_entrypoint.job_function, - pipeline_entrypoint.id, - _job_id=arq_job_id(pipeline_entrypoint), + pipeline, job = await enqueue_pipeline_for_score_set( + db, + redis, + pipeline_name=pipeline_name, + score_set=score_set, + user=user, + extra_params={key: value for key, value in extra_params}, ) + click.echo(f"Created pipeline '{pipeline_name}' (id={pipeline.id}, correlation_id={pipeline.correlation_id})") if job: click.echo(f"Enqueued start_pipeline job: {job.job_id}. Pipeline will execute asynchronously.") else: click.echo("Job was already enqueued (duplicate).", err=True) + except (KeyError, ValueError) as e: + click.echo(f"Failed to create pipeline: {e}", err=True) + sys.exit(1) finally: await redis.aclose() db.close() diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py new file mode 100644 index 000000000..169f17031 --- /dev/null +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -0,0 +1,276 @@ +"""Drive a score-set pipeline (mapping or annotation enrichment) over many score sets at once. + +Two selection modes: + +* ``--select unmapped`` (default) → runs ``map_annotate_score_set`` over score sets that still need + mapping. "Needs mapping" = a variant lacking a **live** ``MappingRecord`` dated on/after + ``--remap-before`` (default 2026-05-08, significant mapping-capability upgrade). This catches + never-mapped, partially-mapped, and *stale-but-complete* score sets — including reshape-backfilled + ones, whose records carry the old ``mapped_date`` and so need a real re-map under the upgraded + pipeline. +* ``--select unenriched`` → runs ``annotate_score_set`` over mapped score sets that have not yet had + the reverse-translation fan-out (no derived, non-authoritative live allele link). This is the + post-backfill enrichment half. + +**Throughput — gene-grouped ordering.** Score sets whose variants resolve to the same CAIDs/PAIDs +cache the expensive ClinGen CAR resolution (24h TTL) and reuse already-deduplicated alleles. So by +default the run is ordered by a normalized gene key — ``coalesce(mapped_hgnc_name, +target_accession.gene, target_gene.name)``, first whitespace token uppercased (``JAG1 Exon 1-7`` → +``JAG1``) — clustering same-gene score sets consecutively so the second-onward sets in a gene group +hit a warm cache and existing alleles. (The resolved ``mapped_hgnc_name`` is null pre-mapping, so the +unmapped mode falls back to the accession gene / submitter name.) ``--no-group-by-gene`` sorts by id. + +Enrichment is non-blocking; both pipelines hit external services, so ``--delay-seconds`` paces starts +and ``--limit`` bounds a batch. Idempotent: re-runs skip score sets already in the target end state +(``--force`` overrides). + +Usage:: + + poetry run python -m mavedb.scripts.run_score_set_pipelines --dry-run + poetry run python -m mavedb.scripts.run_score_set_pipelines --limit 50 --delay-seconds 3 + poetry run python -m mavedb.scripts.run_score_set_pipelines --select unenriched --limit 50 + poetry run python -m mavedb.scripts.run_score_set_pipelines --remap-before 2026-05-08 +""" + +from __future__ import annotations + +import asyncio +import datetime +import logging +from typing import Optional + +import asyncclick as click +from arq import create_pool +from sqlalchemy import and_, not_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.db.session import SessionLocal +from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set +from mavedb.models.enums.processing_state import ProcessingState +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.user import User +from mavedb.models.variant import Variant +from mavedb.worker.settings import RedisWorkerSettings + +logger = logging.getLogger(__name__) + +# The mapping-capability upgrade: score sets last mapped before this need a re-map, even if their +# mapping_state is "complete". +DEFAULT_REMAP_BEFORE = datetime.date(2026, 5, 8) + +# Default pipeline per selection mode. +PIPELINE_FOR_SELECT = { + "unmapped": "map_annotate_score_set", + "unenriched": "annotate_score_set", +} + + +def _normalize_gene(raw: str) -> str: + """Gene key for cache-locality ordering: first whitespace token, uppercased. + + ``"JAG1 Exon 1-7"`` → ``"JAG1"``, ``" brca1 "`` → ``"BRCA1"``. Gene symbols are one token, so the + first word is the gene; upper-casing folds submitter case differences together. + """ + tokens = raw.strip().split() + return tokens[0].upper().strip(".,;:") if tokens else "" + + +def _grouping_key(score_set: ScoreSet) -> str: + """Normalized gene key for a score set — the min across its target genes for determinism. + + Prefers the resolved gene (``mapped_hgnc_name``, populated only post-mapping), then the accession's + gene, then the submitter target name. + """ + keys = [] + for target_gene in score_set.target_genes: + accession_gene = target_gene.target_accession.gene if target_gene.target_accession else None + raw = target_gene.mapped_hgnc_name or accession_gene or target_gene.name + if raw: + keys.append(_normalize_gene(raw)) + return min(keys) if keys else "" + + +def _needs_mapping(remap_before: datetime.date): + """Predicate: the score set has a variant lacking a live MappingRecord dated on/after the cutoff.""" + fresh_record = select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, + MappingRecord.valid_to.is_(None), + MappingRecord.mapped_date >= remap_before, + ) + variant_needing_mapping = select(Variant.id).where(Variant.score_set_id == ScoreSet.id, ~fresh_record.exists()) + return variant_needing_mapping.exists() + + +def _needs_enrichment(): + """Predicate: the score set is mapped (a live MappingRecord) but has no reverse-translation-derived + (non-authoritative) live allele link yet.""" + has_live_record = ( + select(MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(Variant.score_set_id == ScoreSet.id, MappingRecord.valid_to.is_(None)) + ) + has_derived_link = ( + select(MappingRecordAllele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + Variant.score_set_id == ScoreSet.id, + MappingRecord.valid_to.is_(None), + MappingRecordAllele.valid_to.is_(None), + MappingRecordAllele.is_authoritative.is_(False), + ) + ) + return and_(has_live_record.exists(), not_(has_derived_link.exists())) + + +def select_score_sets( + db: Session, + *, + phase: str = "unmapped", + remap_before: datetime.date = DEFAULT_REMAP_BEFORE, + only_published: bool = True, + force: bool = False, + score_set_urn: Optional[str] = None, + group_by_gene: bool = True, + limit: Optional[int] = None, +) -> list[ScoreSet]: + """Return score sets to run, gene-grouped by default. + + ``phase`` picks the eligibility predicate (``"unmapped"`` / ``"unenriched"``); ``force`` bypasses + it (every processed score set qualifies). Only variant-processed score sets are considered (the + pipelines assume variants exist), published unless ``only_published=False``. Ordering: by the + normalized gene key so same-gene sets cluster (``group_by_gene``), else by id; ``limit`` applies + after ordering so whole gene groups are taken first. + """ + query = ( + select(ScoreSet) + .options(selectinload(ScoreSet.target_genes).selectinload(TargetGene.target_accession)) + .where(ScoreSet.processing_state == ProcessingState.success) + ) + if only_published: + query = query.where(ScoreSet.private.is_(False), ScoreSet.published_date.isnot(None)) + if score_set_urn is not None: + query = query.where(ScoreSet.urn == score_set_urn) + if not force: + query = query.where(_needs_mapping(remap_before) if phase == "unmapped" else _needs_enrichment()) + + score_sets = list(db.scalars(query).unique().all()) + score_sets.sort(key=(lambda ss: (_grouping_key(ss), ss.id)) if group_by_gene else (lambda ss: ss.id)) + return score_sets[:limit] if limit is not None else score_sets + + +def _resolve_user(db: Session, score_set: ScoreSet, updater_id: Optional[int]) -> Optional[User]: + """The user to attribute the pipeline to: explicit override, else the score set's modifier/creator.""" + resolved_id = updater_id or score_set.modified_by_id or score_set.created_by_id + if resolved_id is None: + return None + return db.scalars(select(User).where(User.id == resolved_id)).one_or_none() + + +@click.command() +@click.option( + "--select", + "phase", + type=click.Choice(["unmapped", "unenriched"]), + default="unmapped", + show_default=True, + help="Which score sets to run: those needing mapping, or those mapped-but-not-enriched.", +) +@click.option("--pipeline", "pipeline_name", default=None, help="Override the per-mode default pipeline.") +@click.option( + "--remap-before", + "remap_before", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=DEFAULT_REMAP_BEFORE.isoformat(), + show_default=True, + help="Re-map score sets last mapped before this date (unmapped mode only).", +) +@click.option("--score-set-urn", "score_set_urn", default=None, help="Run only this score set.") +@click.option("--updater-id", "updater_id", type=int, default=None, help="User to attribute pipelines to.") +@click.option("--limit", type=int, default=None, help="Max score sets to enqueue this run.") +@click.option( + "--delay-seconds", "delay_seconds", type=float, default=0.0, show_default=True, help="Pause between starts." +) +@click.option("--no-group-by-gene", "group_by_gene", flag_value=False, default=True, help="Sort by id instead of gene.") +@click.option("--force", is_flag=True, help="Ignore the eligibility check (include already-done score sets).") +@click.option("--include-unpublished", is_flag=True, help="Also include private/unpublished score sets.") +@click.option("--dry-run", is_flag=True, help="List the selected score sets without enqueuing anything.") +async def main( + phase: str, + pipeline_name: Optional[str], + remap_before: datetime.datetime, + score_set_urn: Optional[str], + updater_id: Optional[int], + limit: Optional[int], + delay_seconds: float, + group_by_gene: bool, + force: bool, + include_unpublished: bool, + dry_run: bool, +) -> None: + """Run a score-set pipeline over many score sets, gene-grouped for cache reuse.""" + pipeline = pipeline_name or PIPELINE_FOR_SELECT[phase] + db = SessionLocal() + try: + score_sets = select_score_sets( + db, + phase=phase, + remap_before=remap_before.date(), + only_published=not include_unpublished, + force=force, + score_set_urn=score_set_urn, + group_by_gene=group_by_gene, + limit=limit, + ) + click.echo(f"{len(score_sets)} score set(s) selected ({phase}) for pipeline '{pipeline}'.") + + if dry_run: + for score_set in score_sets: + click.echo(f" {_grouping_key(score_set):<12} {score_set.urn}") + click.echo("Dry run — nothing enqueued.") + return + + if not score_sets: + return + + redis = await create_pool(RedisWorkerSettings) + enqueued = 0 + try: + for index, score_set in enumerate(score_sets): + user = _resolve_user(db, score_set, updater_id) + if user is None: + click.echo(f" SKIP {score_set.urn}: no updater found (pass --updater-id).", err=True) + continue + + try: + pipeline_run, job = await enqueue_pipeline_for_score_set( + db, redis, pipeline_name=pipeline, score_set=score_set, user=user + ) + except (KeyError, ValueError) as exc: + click.echo(f" FAIL {score_set.urn}: {exc}", err=True) + continue + + enqueued += 1 + disposition = "queued" if job else "duplicate" + click.echo( + f" [{enqueued}/{len(score_sets)}] {score_set.urn} → pipeline {pipeline_run.id} ({disposition})" + ) + + if delay_seconds and index < len(score_sets) - 1: + await asyncio.sleep(delay_seconds) + + finally: + await redis.aclose() + + click.echo(f"Enqueued {enqueued} pipeline(s).") + + finally: + db.close() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() From 4b640e12a91948868f965b4c53f59be7023ca204 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 14:10:29 -0700 Subject: [PATCH 84/93] fix(variants): accept VariationDescriptor when extracting hgvs from post-mapped vrs get_hgvs_from_post_mapped only matched a bare "Allele" type, so post-mapped payloads wrapped in a VariationDescriptor fell through to the unhandled branch and returned None instead of their hgvs. --- src/mavedb/lib/variants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index 03203e3e7..677eb5de5 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -77,7 +77,7 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any], *, combine_cis: bo if post_mapped_vrs["type"] in ("Haplotype", "CisPhasedBlock"): # type: ignore members = post_mapped_vrs["members"] - elif post_mapped_vrs["type"] == "Allele": # type: ignore + elif post_mapped_vrs["type"] in ("Allele", "VariationDescriptor"): # type: ignore members = [post_mapped_vrs] else: return None From 575a2e26894e7e959a90b86ce8d2ed4263e8fb28 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 17:09:16 -0700 Subject: [PATCH 85/93] feat(migration): add manual backfill of allele substrate from mapped_variants Historical score sets have no mapping_records/alleles data yet, since that substrate is populated only by the live mapping job going forward. This reconstructs it deterministically from existing mapped_variants rows so the new serving/read layer (v_variant_annotations, published_variants MV) resolves for old data too, without redoing the reverse-translation fan-out (deferred to a decoupled enrichment pass). - add migrate_mapped_variants_to_allele_substrate manual migration, with verify/rollback/rebuild-annotations subcommands and idempotent re-run support - freeze the legacy MV-index migration's column signature as a literal instead of importing it from the model, since a later migration rewrites that model - add EventReason.MIGRATED for backfill-reconstructed annotation events, since a migration can't know the original creation history, only present/absent/failed --- ...ate_mapped_variants_to_allele_substrate.py | 1326 +++++++++++++++++ src/mavedb/models/enums/event_reason.py | 9 + 2 files changed, 1335 insertions(+) create mode 100644 alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py diff --git a/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py new file mode 100644 index 000000000..32c752a6b --- /dev/null +++ b/alembic/manual_migrations/migrate_mapped_variants_to_allele_substrate.py @@ -0,0 +1,1326 @@ +"""Backfill the Allele substrate (``mapping_records`` / ``alleles`` / +``mapping_record_alleles`` + the annotation link tables) from existing +``mapped_variants`` rows. + +Context +------- +The annotation/serving layer is migrating off ``MappedVariant`` onto a +parallel-tables model:: + + Variant ─< MappingRecord (1 live/variant, ValidTime) ─< MappingRecordAllele (ValidTime) >─ Allele (dedup by vrs_digest) + +This script populates the new tables from the *existing* ``mapped_variants`` data so +the read-cutover (the ``v_variant_annotations`` view / ``published_variants`` MV +rewrites and the serving readers) resolves for historical score sets, which are +otherwise empty in the new substrate. + +This is the **reshape half of the hybrid migration**: it reconstructs the +authoritative spine deterministically from old data with no external-service +calls. It deliberately does **not** reproduce the reverse-translation +equivalence fan-out (the per-level coding/genomic/protein alleles and their +``projection_group`` pairing) — that data was never computed for old mappings +(``MappedVariant`` holds a single ``post_mapped`` VRS plus cross-level HGVS +*strings*, and an Allele needs a real ``vrs_digest`` that a string cannot +supply). Restoring that richness is the decoupled *enrichment* half: queue the +reverse-translation + annotation-refresh jobs per score set on the worker after +this backfill lands. Serving is correct at the measured level as soon as this +runs; enrichment only adds cross-level breadth and blocks nothing. + +What is reconstructed +--------------------- +* **MappingRecord history** — every ``MappedVariant`` version of a variant with a resolvable sequence + level becomes a ``MappingRecord`` with a gap-free ``[valid_from, valid_to)`` window derived from the + versions' ``mapped_date`` ordering. The ``current=True`` version is the live tail (``valid_to IS + NULL``). History dates are inherently approximate: ``MappedVariant`` records versioning with a + ``current`` boolean, not transaction time, so a window boundary is the successor's ``mapped_date``, + not the exact instant the swap happened. A version whose sequence level can't be attributed is + skipped **individually** (its siblings still get their records — see Idempotency below); if that + version happens to be the live one, the variant has no live record and stays eligible for retry. + Live dates that precede a historical version's are a logged anomaly (a likely re-map candidate), not + a fatal condition — see :func:`_reconstruct_windows`. +* **Authoritative Allele** — from each version's ``post_mapped`` VRS, deduplicated by ``vrs_digest`` + against an in-memory, per-chunk cache (:func:`_authoritative_allele` — not the live job's + ``get_or_create_allele``, which queries per call; see :func:`do_migration`'s docstring for why). + Carries the CAID and the assay-level HGVS + across so the allele-keyed serving endpoints resolve. The assay-level HGVS itself is read through a + three-tier fallback (:func:`_hgvs_assay_level_with_source`): ``mv.hgvs_assay_level`` directly, else + the per-level column matching the resolved level (``hgvs_g``/``hgvs_c``/``hgvs_p`` — backfilled later + and never caught up on a large share of old data), else the ``post_mapped`` VRS blob's own + ``expressions`` (the mapper sometimes stamps or reconstructs an HGVS expression directly on the VRS + Allele even when neither dedicated column got populated). Even with all three tiers, some legacy rows + carry no recoverable HGVS at all — that gap is real and needs the mapper re-run to close, not another + migration-time trick. Reading only ``hgvs_assay_level`` (as this script originally did) left most + reconstructed Alleles (and ``MappingRecord.hgvs_assay_level``, and therefore its derived + ``transcript``) NULL even when a real string was recoverable — silently starving any downstream pass + that resolves a transcript from it. +* **MappingRecordAllele** — one ``is_authoritative=True`` link per record on the + record's validity window. ``projection_group`` stays NULL (no fan-out). +* **Annotation timelines** — reconstructed across *every* mapping version, not just the live one: + ``gnomad_variants`` → ``gnomad_allele_links``, ``clinical_controls`` → ``clinvar_allele_links``, + VEP columns → ``vep_allele_consequences``. Each mapping version's bundled annotations are captured + against that version's window and, per allele, merged per source key (gnomAD variant / ClinVar + control / consequence value) into maximal intervals — so a continuously-held annotation is one link + dated from its **earliest** mapping version, open-ended only if it reaches the live record. A + release change that coincided with a re-map surfaces as a superseding closed+live pair. This is + faithful (the old jobs wrote annotations on the current mapping row, so their changes aligned to + re-map boundaries), just **coarse** — sub-mapping-version churn collapses to the latest value in + that window, and boundary timestamps are day-grained. The valid-time axis is uniformly the mapping + date across sources (VEP's real ``access_date`` is kept in its own audit column, not as + ``valid_from``, so ``as_of`` stays coherent). VEP has no Ensembl release string in old data, but + ``access_date`` is enough to derive one: ``source_version`` is looked up from a snapshotted table of + Ensembl release dates (:data:`_ENSEMBL_VEP_RELEASE_DATES`), not stamped as an opaque sentinel — see + :func:`_resolve_vep_source_version`. Only a missing ``access_date``, or one older than the table's + earliest known release, falls back to the ``"legacy"`` sentinel. + + **Exception:** a gnomAD or ClinVar window that would otherwise land live on a **protein-level** allele + is force-closed at the migration's own run time instead (VEP is unaffected — see below). The old + model could proxy both onto protein-level rows because there was nowhere else to attach them; the new + model resolves them correctly through the CA/PA link graph onto nucleotide siblings instead + (``lib/allele_measurements.py``), so carrying the proxy forward live would just create a second, + competing provenance story for the same value. Applied uniformly even though gnomAD is not known to + have actually been proxied this way in practice (ClinVar was) — it is the same underlying anti-pattern + (a nucleotide-level concept attached to a protein-level allele), so the guard costs nothing to keep + general. The historical fact is kept (closed, not deleted) for point-in-time audit; live annotations + for these alleles return once cross-level annotation fan-out re-derives them properly. + +Idempotency / no double-write +----------------------------- +A variant is either old-substrate or new (the parallel-tables invariant); native new-model variants +have no ``mapped_variants``. The migration therefore processes only variants that have +``mapped_variants`` and **no live** ``MappingRecord`` yet — live, not "any", because a variant whose +live mapped_variant version has no resolvable sequence level is left with only historical records (see +above) and must stay selectable so a later re-map or QC fix can complete it. Within a selected variant, +each window is additionally guarded by a ``(variant_id, mapped_date)`` existence check, so a re-run +that revisits a partially-migrated variant only fills in the windows still missing rather than +duplicating the ones a prior pass already wrote. Annotation links are existence-checked before +insertion, so a partial run resumes cleanly; run ``rebuild-annotations`` after resuming a +partially-migrated variant to pick up any timelines the skipped windows didn't get to build. + +Standalone convention +--------------------- +Follows the ``migrate_jsonb_ranges_to_table_rows.py`` pattern: invoked outside +``alembic upgrade`` (it can take a long time on production data), commits in +variant batches, and offers ``verify`` / ``rollback`` commands. + +Usage +----- +:: + + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate # run + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate verify # inspect without writing + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rollback # remove backfilled records (destructive) + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate rebuild-annotations # (re)build annotation links for already-migrated variants + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --score-set urn:mavedb:00000001-a-1 + python -m alembic.manual_migrations.migrate_mapped_variants_to_allele_substrate migrate --batch-size 1000 --dry-run +""" + +from __future__ import annotations + +import argparse +import logging +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date, datetime, time, timezone +from time import perf_counter # `time` above is datetime.time; import the timer directly +from typing import Optional + +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, configure_mappers + +from mavedb.models import * # noqa: F401,F403 pylint: disable=wildcard-import — register all mappers for configure_mappers() +from mavedb.db.session import SessionLocal +from mavedb.lib.annotation_status_manager import AnnotationStatusManager +from mavedb.lib.variants import get_hgvs_from_post_mapped +from mavedb.models.allele import Allele +from mavedb.models.annotation_event import AnnotationEvent +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink +from mavedb.models.enums.annotation_type import AnnotationType +from mavedb.models.enums.disposition import Disposition +from mavedb.models.enums.event_reason import EventReason +from mavedb.models.enums.sequence_level import SequenceLevel +from mavedb.models.gnomad_allele_link import GnomadAlleleLink +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence + +configure_mappers() + +logger = logging.getLogger(__name__) + +# Old VEP data carries no Ensembl release string, only an access_date — but source_version is derivable: +# Ensembl releases (and VEP's version with them) are coordinated, dated, and public +# (https://github.com/Ensembl/ensembl-vep/releases), so the release active on a given access_date can +# be looked up rather than stamped as an opaque sentinel. Each entry is a release's *earliest* GitHub +# tag date (patch tags like 92.1-92.6 are hotfixes within release 92, not new majors); an access_date +# is attributed to the newest release whose date is <= it. Snapshotted from the GitHub API on +# 2026-07-20 (covers releases 92 through 116); this table does not self-update, so re-run the same +# query and extend it before backfilling data annotated under a release newer than the last entry here. +_ENSEMBL_VEP_RELEASE_DATES: list[tuple[date, str]] = [ + (date(2018, 4, 12), "92"), + (date(2018, 7, 17), "93"), + (date(2018, 10, 2), "94"), + (date(2019, 1, 10), "95"), + (date(2019, 4, 8), "96"), + (date(2019, 7, 3), "97"), + (date(2019, 9, 27), "98"), + (date(2020, 1, 16), "99"), + (date(2020, 4, 29), "100"), + (date(2020, 8, 20), "101"), + (date(2020, 11, 30), "102"), + (date(2021, 2, 16), "103"), + (date(2021, 5, 5), "104"), + (date(2021, 12, 9), "105"), + (date(2022, 4, 14), "106"), + (date(2022, 7, 13), "107"), + (date(2022, 10, 20), "108"), + (date(2023, 2, 8), "109"), + (date(2023, 7, 18), "110"), + (date(2024, 1, 11), "111"), + (date(2024, 5, 16), "112"), + (date(2024, 10, 18), "113"), + (date(2025, 5, 8), "114"), + (date(2025, 9, 3), "115"), + (date(2026, 6, 10), "116"), +] + +# Fallback sentinel for the cases the release-date table can't resolve: no access_date at all, or an +# access_date older than the table's earliest known release (GitHub has no tags before release/92, so +# anything before 2018-04-12 is out of range). VepAlleleConsequence.source_version is NOT NULL, and a +# refresh run should treat this value as unversioned and re-confirm it against the current release +# rather than trusting it as current — the same behavior the sentinel always had. +LEGACY_VEP_SOURCE_VERSION = "legacy" + + +def _resolve_vep_source_version(access_date: Optional[date]) -> str: + """Map a legacy VEP ``access_date`` to the Ensembl release active on that day. + + Returns :data:`LEGACY_VEP_SOURCE_VERSION` when ``access_date`` is missing or predates the earliest + release the table knows about — both cases where the release genuinely can't be determined, as + opposed to just not being in the table's covered range for lack of maintenance. + """ + if access_date is None or access_date < _ENSEMBL_VEP_RELEASE_DATES[0][0]: + return LEGACY_VEP_SOURCE_VERSION + resolved = _ENSEMBL_VEP_RELEASE_DATES[0][1] + for release_date, version in _ENSEMBL_VEP_RELEASE_DATES: + if release_date > access_date: + break + resolved = version + return resolved + + +def _as_dt(d: date) -> datetime: + """Lift a ``date`` to a timezone-aware ``datetime`` for a ValidTime boundary. + + ValidTime windows are ``DateTime(timezone=True)``; ``MappedVariant`` versioning + is day-grained (``mapped_date``), so midnight UTC is the reconstructed boundary. + """ + return datetime.combine(d, time.min, tzinfo=timezone.utc) + + +def _resolve_level(mv: MappedVariant) -> Optional[SequenceLevel]: + """The sequence level a ``MappedVariant`` was assayed/aligned at. + + ``alignment_level`` is populated on every row by the target-gene-mapping QC + backfill and is the direct source (the mapping job sets a record's + ``assay_level`` and ``alignment_level`` to the same value). For the rare + legacy row the QC backfill could not attribute, fall back to whichever mapped + HGVS column is populated. Returns ``None`` when neither signal is present — + the record cannot be created (``assay_level`` is NOT NULL) and the caller + skips the variant with a logged anomaly. + """ + if mv.alignment_level is not None: + return mv.alignment_level + if mv.hgvs_g: + return SequenceLevel.genomic + if mv.hgvs_c: + return SequenceLevel.cdna + if mv.hgvs_p: + return SequenceLevel.protein + return None + + +_LEVEL_HGVS_COLUMN = { + SequenceLevel.genomic: "hgvs_g", + SequenceLevel.cdna: "hgvs_c", + SequenceLevel.protein: "hgvs_p", +} + + +def _hgvs_assay_level_with_source(mv: MappedVariant, level: SequenceLevel) -> tuple[Optional[str], str]: + """Resolve the assay-level HGVS string, reporting which of three tiers supplied it. + + ``mv.hgvs_assay_level`` is frequently ``NULL`` on legacy rows — it was backfilled later and never + caught up on a large share of old data — and reading only it (as this script originally did) meant + most reconstructed Alleles got a NULL ``hgvs_g``/``hgvs_c``/``hgvs_p`` and + ``MappingRecord.hgvs_assay_level`` even when the real string was recoverable. Since + ``MappingRecord.transcript`` derives from ``hgvs_assay_level``, that silently starved any downstream + pass that resolves a transcript from it (e.g. cross-level annotation fan-out). + + Three tiers, in order: + + 1. ``"direct"`` — ``mv.hgvs_assay_level`` itself. + 2. ``"level_column"`` — the per-level column matching ``level`` (``hgvs_g``/``hgvs_c``/``hgvs_p``), + mirroring :func:`_resolve_level`'s own fallback (same column), just for the *value* rather than + the level. + 3. ``"vrs_expression"`` — the ``post_mapped`` VRS blob's own ``expressions``. Some legacy rows have + *neither* dedicated column populated, but the mapper stamped an HGVS expression directly onto + the VRS Allele it produced (``dcd_mapping`` sets this unconditionally for coding-layer alleles, + and reconstructs it for genomic/protein when accession resolution succeeds — so it is not + universal either, just an independent, additional chance). Reuses + :func:`mavedb.lib.variants.get_hgvs_from_post_mapped`, the same extraction serving already relies + on, rather than re-parsing VRS JSON here. A malformed or VRS-1.x blob raises there; caught here + and treated as "no expression" so one bad blob doesn't abort the whole variant. + + Returns ``(None, "none")`` when no tier resolves anything — some legacy data genuinely carries no + recoverable HGVS at all, and that gap cannot be closed without re-running the mapper. + """ + if mv.hgvs_assay_level: + return mv.hgvs_assay_level, "direct" + + level_value = getattr(mv, _LEVEL_HGVS_COLUMN[level]) + if level_value: + return level_value, "level_column" + + try: + vrs_value = get_hgvs_from_post_mapped(mv.post_mapped) + except (KeyError, ValueError): + vrs_value = None + if vrs_value: + return vrs_value, "vrs_expression" + + return None, "none" + + +def _resolve_hgvs_assay_level(mv: MappedVariant, level: SequenceLevel) -> Optional[str]: + """The assay-level HGVS string for a version — see :func:`_hgvs_assay_level_with_source`.""" + return _hgvs_assay_level_with_source(mv, level)[0] + + +@dataclass +class _WindowResult: + """``_reconstruct_windows``'s output: the ordered windows plus an anomaly flag. + + Kept as one return value (rather than ``None``-means-skip on windows, anomaly bundled in) so a + caller can't forget to check the flag the way a bare tuple invites. + """ + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] + live_precedes_history: bool + + +def _reconstruct_windows(mvs: list[MappedVariant]) -> Optional[_WindowResult]: + """Order a variant's ``MappedVariant`` versions and assign ValidTime windows. + + Returns ``(mv, valid_from, valid_to)`` per version, gap-free: each version's + ``valid_to`` is its successor's ``valid_from``, and the live tail's is + ``None``. The ``current=True`` version is the live tail regardless of its + ``mapped_date``. Returns ``None`` (skip) when no version is current — a + variant with mappings but no live one is a source anomaly we do not guess a + live state for. + + ``live_precedes_history=True`` flags the case where the live version's ``mapped_date`` is earlier + than some historical version's — a data-quality anomaly (``current`` should always be the newest). + When it happens, the pairwise clamp below still prevents inverted windows, but at the cost of + collapsing the affected historical window to zero-width (invisible to ``as_of``) rather than + losing data outright: the caller surfaces the flag as a loud, variant-attributed warning instead + of silently reordering, since it likely means the source variant wants a re-map. + """ + current = [m for m in mvs if m.current] + history = [m for m in mvs if not m.current] + + if not current: + return None + if len(current) == 1: + live = current[0] + else: + # Multiple live rows is a source anomaly; keep the newest live and demote + # the rest into history so no data is dropped. + current.sort(key=lambda m: (m.mapped_date, m.id)) + live = current[-1] + history.extend(current[:-1]) + logger.warning("Variant %s has %d current mapped_variants; keeping newest live.", live.variant_id, len(current)) + + history.sort(key=lambda m: (m.mapped_date, m.id)) + ordered = history + [live] + live_precedes_history = bool(history) and live.mapped_date < history[-1].mapped_date + + windows: list[tuple[MappedVariant, datetime, Optional[datetime]]] = [] + for i, m in enumerate(ordered): + valid_from = _as_dt(m.mapped_date) + valid_to: Optional[datetime] = None + if i + 1 < len(ordered): + successor = _as_dt(ordered[i + 1].mapped_date) + # Same-day or out-of-order successor: clamp so the window is never + # inverted (a zero-width [t, t) window is simply invisible to as_of). + valid_to = max(successor, valid_from) + windows.append((m, valid_from, valid_to)) + return _WindowResult(windows=windows, live_precedes_history=live_precedes_history) + + +def _fill_missing_allele_fields(allele: Allele, mv: MappedVariant, level: SequenceLevel) -> None: + """Backfill identity fields onto a (possibly pre-existing, deduped) allele. + + Dedup means the first variant to mint an allele sets its fields; a later + variant sharing the ``vrs_digest`` may carry the CAID or level HGVS the first + lacked. Those are content-derived (deterministic from the allele), so + fill-if-missing is safe and never overwrites a populated value. + """ + if allele.clingen_allele_id is None and mv.clingen_allele_id: + allele.clingen_allele_id = mv.clingen_allele_id + + hgvs_col = _LEVEL_HGVS_COLUMN[level] + hgvs_value = _resolve_hgvs_assay_level(mv, level) + if getattr(allele, hgvs_col) is None and hgvs_value: + setattr(allele, hgvs_col, hgvs_value) + + +def _authoritative_allele( + db: Session, mv: MappedVariant, level: SequenceLevel, allele_cache: dict[str, Allele] +) -> Optional[Allele]: + """Get-or-create the authoritative allele from ``mv.post_mapped``, via an in-memory dedup cache. + + Returns ``None`` for versions with no post-mapped representation (failed or benign-absent variants + get a record but no linked allele, matching the mapping job). + + Unlike :func:`mavedb.lib.variant_translations.get_or_create_allele` (used by the live mapping job), + this does **not** query the database or flush per call. ``allele_cache`` is pre-seeded by the + caller with one bulk ``vrs_digest IN (...)`` query per chunk (see :func:`do_migration`), and this + function keeps it updated in-memory as new alleles are drafted — so a digest shared by two windows + in the *same* chunk (a common dedup case) resolves from the dict, not a second round trip. This is + safe only because the session runs with ``autoflush=False`` (a query wouldn't see an unflushed + insert anyway) and because the caller flushes once per variant, not per window: an actual + ``uq_alleles_vrs_digest`` collision (e.g. a concurrent live mapping run) still surfaces at that + flush, inside the same per-variant SAVEPOINT it always has — this only removes the redundant reads, + not the write-time integrity check. + """ + post_mapped = mv.post_mapped or {} + digest = post_mapped.get("id") + if not digest: + return None + + existing = allele_cache.get(digest) + if existing is not None: + _fill_missing_allele_fields(existing, mv, level) + return existing + + hgvs_value = _resolve_hgvs_assay_level(mv, level) + allele = Allele( + vrs_digest=digest, + level=level, + hgvs_g=hgvs_value if level == SequenceLevel.genomic else None, + hgvs_c=hgvs_value if level == SequenceLevel.cdna else None, + hgvs_p=hgvs_value if level == SequenceLevel.protein else None, + clingen_allele_id=mv.clingen_allele_id, + post_mapped=post_mapped, + ) + db.add(allele) + allele_cache[digest] = allele + return allele + + +@dataclass +class _AnnotationObservation: + """One mapping version's annotation payload for an authoritative allele, tagged with that + version's validity window. + + Built fresh from the DB per allele-batch (:func:`_observations_for_alleles`) rather than collected + inline during record creation — so a deduped allele shared by several variants, or spanning a + variant's re-map history, still yields one merged timeline rather than per-version fragments (see + :func:`_write_annotation_timelines`), without needing every observation for the whole run held in + memory at once. + """ + + valid_from: datetime + valid_to: Optional[datetime] + gnomad_variant_ids: tuple[int, ...] + clinvar_control_ids: tuple[int, ...] + vep_consequence: Optional[str] + vep_access_date: Optional[date] + + +def _observation_for( + mv: MappedVariant, valid_from: datetime, valid_to: Optional[datetime] +) -> Optional[_AnnotationObservation]: + """Capture a mapping version's annotations for its window, or ``None`` if it carries none.""" + gnomad_ids = tuple(g.id for g in mv.gnomad_variants) + clinvar_ids = tuple(c.id for c in mv.clinical_controls) + if not gnomad_ids and not clinvar_ids and not mv.vep_functional_consequence: + return None + return _AnnotationObservation( + valid_from=valid_from, + valid_to=valid_to, + gnomad_variant_ids=gnomad_ids, + clinvar_control_ids=clinvar_ids, + vep_consequence=mv.vep_functional_consequence, + vep_access_date=mv.vep_access_date, + ) + + +def _merge_intervals( + windows: list[tuple[datetime, Optional[datetime]]], +) -> list[tuple[datetime, Optional[datetime]]]: + """Merge overlapping/adjacent ``[valid_from, valid_to)`` windows; ``valid_to=None`` is open-ended. + + An annotation held across a run of mapping versions produces contiguous windows (and overlapping + ones when a deduped allele is shared by several variants). Merging collapses each continuous run + into one link — ``valid_from`` = its earliest mapping date, ``valid_to`` open only if the run + reaches the live record — and closes a link only where the annotation actually lapses, rather than + fabricating a boundary at every re-map. + """ + ordered = sorted(windows, key=lambda w: w[0]) + merged: list[tuple[datetime, Optional[datetime]]] = [] + cur_from, cur_to = ordered[0] + for vf, vt in ordered[1:]: + if cur_to is None or vf <= cur_to: # overlap/adjacency; an open-ended window absorbs the rest + cur_to = None if (cur_to is None or vt is None) else max(cur_to, vt) + else: + merged.append((cur_from, cur_to)) + cur_from, cur_to = vf, vt + merged.append((cur_from, cur_to)) + return merged + + +def _allele_has_live_link(db: Session, model: type, allele_id: int) -> bool: + """Whether ``allele_id`` already has a live (``valid_to IS NULL``) link in ``model``. + + Sees **committed** rows only — the session runs ``autoflush=False``, so this does not observe + not-yet-flushed adds from the current pass. It therefore catches prior-run and concurrently- + committed live links (e.g. from the mapping pipeline running alongside this backfill); single-live + within one pass is enforced separately, in memory, by :func:`_write_allele_timeline`. + """ + return db.scalar(sa.select(model.id).where(model.allele_id == allele_id, model.valid_to.is_(None))) is not None + + +def _emit_gnomad_link( + db: Session, + allele_id: int, + gnomad_variant_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(GnomadAlleleLink.id).where( + GnomadAlleleLink.allele_id == allele_id, + GnomadAlleleLink.gnomad_variant_id == gnomad_variant_id, + GnomadAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + db.add( + GnomadAlleleLink( + allele_id=allele_id, gnomad_variant_id=gnomad_variant_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["gnomad_links"] += 1 + + +def _emit_clinvar_link( + db: Session, + allele_id: int, + clinvar_control_id: int, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(ClinvarAlleleLink.id).where( + ClinvarAlleleLink.allele_id == allele_id, + ClinvarAlleleLink.clinvar_control_id == clinvar_control_id, + ClinvarAlleleLink.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run; ClinVar is multi-live per (allele, control) + return + db.add( + ClinvarAlleleLink( + allele_id=allele_id, clinvar_control_id=clinvar_control_id, valid_from=valid_from, valid_to=valid_to + ) + ) + stats["clinvar_links"] += 1 + + +def _emit_vep_consequence( + db: Session, + allele_id: int, + consequence: str, + access_date: date, + valid_from: datetime, + valid_to: Optional[datetime], + stats: Counter, +) -> None: + already = db.scalar( + sa.select(VepAlleleConsequence.id).where( + VepAlleleConsequence.allele_id == allele_id, + VepAlleleConsequence.functional_consequence == consequence, + VepAlleleConsequence.valid_from == valid_from, + ) + ) + if already is not None: # idempotent re-run + return + source_version = _resolve_vep_source_version(access_date) + stats["vep_source_version_legacy_fallback" if source_version == LEGACY_VEP_SOURCE_VERSION else "vep_source_version_resolved"] += 1 + db.add( + VepAlleleConsequence( + allele_id=allele_id, + functional_consequence=consequence, + source_version=source_version, + access_date=access_date, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["vep_consequences"] += 1 + + +def _write_allele_timeline( + db: Session, allele_id: int, obs_list: list[_AnnotationObservation], stats: Counter, run_at: datetime +) -> None: + """Build and emit one allele's merged annotation link timelines (no flush/commit here). + + Keyed per source — gnomAD by ``gnomad_variant_id``, ClinVar by ``clinvar_control_id``, VEP by + consequence value — each key's windows are merged so a continuously-held annotation is a single + link dated from its earliest mapping version, open-ended only if it reaches the live record. + + **Single-live-per-allele** for gnomAD/VEP is enforced with an in-memory flag seeded from committed + state (:func:`_allele_has_live_link`). This is the fix for the ``autoflush=False`` session: a + deduped allele linked to two *different* gnomAD variants (from two variants sharing the allele) + would otherwise emit two live links and violate ``uq_gnomad_allele_links_live`` at flush, since the + per-add DB check can't see the earlier unflushed add. The flag also folds in a link already live + from a prior run or the concurrent mapping pipeline. ClinVar is multi-live per ``(allele, control)``. + + **Protein-level gnomAD/ClinVar are a legacy proxy the new model doesn't carry forward live.** The + old model could attach ClinVar significance and gnomAD frequency to protein-level ``MappedVariant`` + rows as a proxy — there was nowhere else to put them. Both are intrinsically nucleotide-level + concepts (a ClinVar submission and a gnomAD allele frequency are both keyed to a genomic variant, + not a protein change), and the new model resolves them correctly for a protein allele via the CA/PA + link graph onto its nucleotide siblings (see ``lib/allele_measurements.py``), not by direct + attachment. A live direct link here would just be a second, competing provenance story for the same + value. So a protein allele's gnomAD/ClinVar window is force-closed at ``run_at`` (the migration's + own run time) instead of staying open-ended: the historical fact that this had been proxy-annotated + is kept for point-in-time audit, but it stops being *live* data at the moment of reshape, to be + replaced once cross-level annotation fan-out re-derives it correctly through the link graph. VEP is + exempt — a functional consequence is computed relative to the assayed representation itself, so it + is not a proxied nucleotide-level value the way gnomAD/ClinVar are. + """ + gnomad_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + clinvar_windows: dict[int, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_windows: dict[str, list[tuple[datetime, Optional[datetime]]]] = defaultdict(list) + vep_access: dict[str, list[date]] = defaultdict(list) + + for obs in obs_list: + for gnomad_variant_id in obs.gnomad_variant_ids: + gnomad_windows[gnomad_variant_id].append((obs.valid_from, obs.valid_to)) + for clinvar_control_id in obs.clinvar_control_ids: + clinvar_windows[clinvar_control_id].append((obs.valid_from, obs.valid_to)) + if obs.vep_consequence: + vep_windows[obs.vep_consequence].append((obs.valid_from, obs.valid_to)) + if obs.vep_access_date is not None: + vep_access[obs.vep_consequence].append(obs.vep_access_date) + + is_protein = False + if gnomad_windows or clinvar_windows: + allele_level = db.scalar(sa.select(Allele.level).where(Allele.id == allele_id)) + is_protein = allele_level == SequenceLevel.protein.value + + gnomad_live_taken = _allele_has_live_link(db, GnomadAlleleLink, allele_id) + for gnomad_variant_id, windows in gnomad_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + # Force-closed, not left live: not subject to the single-live guard below, since it + # never becomes a live row that could collide with uq_gnomad_allele_links_live. + valid_to = run_at + stats["protein_gnomad_links_closed"] += 1 + elif valid_to is None: + if gnomad_live_taken: + logger.warning( + "Allele %s already has a live gnomAD link; skipping gnomad_variant %s.", + allele_id, + gnomad_variant_id, + ) + continue + gnomad_live_taken = True + _emit_gnomad_link(db, allele_id, gnomad_variant_id, valid_from, valid_to, stats) + + for clinvar_control_id, windows in clinvar_windows.items(): + for valid_from, valid_to in _merge_intervals(windows): + if is_protein and valid_to is None: + valid_to = run_at + stats["protein_clinvar_links_closed"] += 1 + _emit_clinvar_link(db, allele_id, clinvar_control_id, valid_from, valid_to, stats) + + vep_live_taken = _allele_has_live_link(db, VepAlleleConsequence, allele_id) + for consequence, windows in vep_windows.items(): + observed_dates = vep_access[consequence] + for valid_from, valid_to in _merge_intervals(windows): + if valid_to is None: + if vep_live_taken: + logger.warning("Allele %s already has a live VEP consequence; skipping %r.", allele_id, consequence) + continue + vep_live_taken = True + access_date = min(observed_dates) if observed_dates else valid_from.date() + _emit_vep_consequence(db, allele_id, consequence, access_date, valid_from, valid_to, stats) + + +def _write_annotation_timelines( + db: Session, observations: dict[int, list[_AnnotationObservation]], stats: Counter, run_at: datetime +) -> None: + """Write each allele's annotation timelines, isolating conflicts per allele. + + Each allele is built + flushed inside its own SAVEPOINT: if a live-uniqueness collision still + slips through the in-memory guard — e.g. the concurrently-running mapping pipeline commits a live + link for a shared allele between the guard's committed-state read and this flush — only that + allele's links roll back (counted as ``annotation_conflicts``), and the rest of the pass proceeds + instead of the whole final commit aborting. + + ``run_at`` is a single timestamp for the whole pass (not re-read per allele), so every protein-level + ClinVar link force-closed by :func:`_write_allele_timeline` in this run shares the same cutover + instant rather than drifting with wall-clock time across a long batch. + """ + for allele_id, obs_list in observations.items(): + try: + with db.begin_nested(): + _write_allele_timeline(db, allele_id, obs_list, stats, run_at) + db.flush() # surface any live-uniqueness violation inside this allele's savepoint + except IntegrityError: + stats["annotation_conflicts"] += 1 + logger.warning("Annotation links for allele %s conflicted (live elsewhere); skipped.", allele_id) + + +def _observations_for_alleles(db: Session, allele_ids: list[int]) -> dict[int, list[_AnnotationObservation]]: + """Rebuild annotation observations for one batch of alleles, from fresh DB state. + + Scoped by allele id, not by variant/chunk: the merge :func:`_write_allele_timeline` performs is only + ever correct once *every* mapping version that ever authoritatively linked to a given allele has been + observed, and alleles are shared arbitrarily across variants — a variant-id chunk boundary says + nothing about which alleles are "done" (an allele's contributing variants can land in different + chunks). Batching by allele id instead means every batch's observations are complete for the alleles + in it, however many separate variant-chunks originally fed those alleles. + + Pairs each authoritative ``MappingRecord`` to its source ``MappedVariant`` version by + ``(variant_id, mapped_date)`` — the record was created with that version's ``mapped_date`` — mirroring + the pairing :func:`_migrate_variant` used when the record was first built. + """ + links = db.execute( + sa.select( + MappingRecordAllele.allele_id, + MappingRecord.variant_id, + MappingRecord.mapped_date, + MappingRecord.valid_from, + MappingRecord.valid_to, + ) + .join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .where(MappingRecordAllele.allele_id.in_(allele_ids), MappingRecordAllele.is_authoritative) + ).all() + if not links: + return {} + + variant_ids = {variant_id for _, variant_id, _, _, _ in links} + mv_by_key: dict[tuple[int, date], MappedVariant] = { + (mv.variant_id, mv.mapped_date): mv + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(variant_ids))).all() + } + + observations: dict[int, list[_AnnotationObservation]] = defaultdict(list) + for allele_id, variant_id, mapped_date, valid_from, valid_to in links: + mv = mv_by_key.get((variant_id, mapped_date)) + if mv is None: + continue # authoritative link survives independently of its source mapped_variant row + observation = _observation_for(mv, valid_from, valid_to) + if observation is not None: + observations[allele_id].append(observation) + return observations + + +def _backfill_annotation_timelines( + db: Session, allele_ids: list[int], *, batch_size: int, run_at: datetime, stats: Counter +) -> None: + """Build and commit annotation link timelines for ``allele_ids``, batch by batch. + + Each batch re-reads its own observations fresh from the DB (:func:`_observations_for_alleles`) and + commits before moving to the next, so memory is bounded by ``batch_size`` alleles at a time + regardless of how many millions of variants fed into ``allele_ids``. This replaces the previous + design, which accumulated every ``_AnnotationObservation`` for the *whole run* in memory before + writing anything at the very end — correct, but the accumulator's unbounded growth over a + multi-million-variant run put steadily increasing pressure on the GC, degrading throughput well + before the run completed (not just a flat per-call cost, but one that compounded with how much had + already been processed). + """ + total = len(allele_ids) + pass_start = perf_counter() + for offset in range(0, total, batch_size): + chunk = allele_ids[offset : offset + batch_size] + observations = _observations_for_alleles(db, chunk) + if observations: + _write_annotation_timelines(db, observations, stats, run_at) + db.commit() + processed = min(offset + batch_size, total) + elapsed = perf_counter() - pass_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Built timelines for {processed}/{total} allele(s)... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + +def _migrate_variant( + db: Session, + variant_id: int, + mvs: list[MappedVariant], + stats: Counter, + skipped: dict[str, list[int]], + annotation_manager: AnnotationStatusManager, + existing_records: set[tuple[int, date]], + allele_cache: dict[str, Allele], +) -> set[int]: + """Reconstruct as much of a variant's record history as its data supports, returning the ids of + every allele this variant authoritatively touched. + + Also records one ``VRS_MAPPING`` event per version created (variant-subject) and one + ``CLINGEN_ALLELE_ID`` / ``MAPPED_HGVS`` event per allele-touch (allele-subject), all reason + :data:`EventReason.MIGRATED` — see :func:`do_migration`'s docstring for why this is scoped to the + mapping/identity side only, not gnomAD/ClinVar/VEP. + + The caller only accumulates these bare ids (not the annotation payload itself) into a global, + whole-run set — a dedup key is all a deferred, allele-batched annotation pass + (:func:`_backfill_annotation_timelines`) needs to know which alleles to revisit; the actual + gnomAD/ClinVar/VEP observations are re-read fresh from the DB per allele-batch there, not carried + in memory from here. Carrying full ``_AnnotationObservation`` payloads for the whole run was the + previous design and is what overloaded the heap at scale — see that function's docstring. Returns + an empty set for a variant with no current version at all — that one is a full skip, since there is + no live state to reconstruct. + + A version with no resolvable sequence level is skipped **individually**, not the whole variant: + ``ValidTime`` history is exactly the mechanism for retaining partial provenance, so one bad version + should not cost its siblings their records. A window's ``valid_from``/``valid_to`` only depend on + its neighbors' ``mapped_date`` (computed up front in :func:`_reconstruct_windows`), never on + whether a record actually got created for those neighbors — so skipping a window here doesn't + distort the boundaries around it. Already-migrated windows (from a prior partial run that + stopped here before this fix, or a live-level failure that keeps this variant eligible for retry — + see :func:`_variant_ids_to_migrate`) are detected via ``existing_records`` (bulk pre-fetched once per + chunk by the caller — see :func:`do_migration`) and skipped as a no-op, so a re-run is safe to + resume mid-variant. Their annotations are not re-derived here; run ``rebuild-annotations`` after a + resumed run to backfill any timelines missed by the skip. + + Every ``MappingRecord``/``Allele``/``MappingRecordAllele`` this variant needs is only ``db.add``'d + here, not flushed, until one ``db.flush()`` at the end covering the whole variant — cut from one + flush per *window* (this was the dominant cost at scale: round trips, not query time, were the + bottleneck). ``MappingRecordAllele`` is built from the ``record``/``allele`` *objects*, not their + ``.id``, specifically so it doesn't need them flushed first — SQLAlchemy resolves the FKs at flush + time. Observations and ``MIGRATED`` events (which need real ``allele.id``/``variant_id`` values) are + built in a second pass over ``pending`` *after* that single flush, once ids exist. + + Skips/anomalies are recorded into ``skipped`` (reason → variant ids), not logged per variant: a + single unmappable score set (e.g. a large pairwise-haplotype library the mapper never converted) + can contribute hundreds of thousands of skips, so :func:`do_migration` rolls them up into one line + per score set at the end rather than emitting a warning each. + """ + result = _reconstruct_windows(mvs) + if result is None: + stats["variants_skipped_no_current"] += 1 + skipped["no_current"].append(variant_id) + logger.debug("Variant %s has mapped_variants but none current; skipped.", variant_id) + return set() + + if result.live_precedes_history: + # Not fatal — the clamp in _reconstruct_windows keeps windows non-inverted — but it silently + # collapses a historical window to zero-width, so surface it loudly: this shape means the + # live mapped_variant's mapped_date is older than a superseded one, which shouldn't happen + # absent a source data-quality issue, and likely wants a re-map to fix at the source. + stats["variants_live_precedes_history"] += 1 + skipped["live_precedes_history"].append(variant_id) + + # (mv, record, allele-or-None, hgvs_assay_level, valid_from, valid_to) per window actually created + # this pass — resolved to real ids by one flush below, then walked again to emit events and collect + # touched allele ids. + pending: list[tuple[MappedVariant, MappingRecord, Optional[Allele], Optional[str], datetime, Optional[datetime]]] = [] + + for mv, valid_from, valid_to in result.windows: + if (variant_id, mv.mapped_date) in existing_records: + continue # already migrated in a prior pass over this variant; resuming, not redoing + + level = _resolve_level(mv) + if level is None: + is_live = valid_to is None + reason = "no_level_live" if is_live else "no_level_historical" + stats[f"windows_skipped_{reason}"] += 1 + skipped[reason].append(variant_id) + logger.debug( + "Variant %s has an unattributable mapped_variant (id=%s, live=%s); window skipped.", + variant_id, + mv.id, + is_live, + ) + continue + + hgvs_assay_level, hgvs_source = _hgvs_assay_level_with_source(mv, level) + if hgvs_source == "level_column": + stats["hgvs_recovered_from_level_column"] += 1 + elif hgvs_source == "vrs_expression": + stats["hgvs_recovered_from_vrs_expression"] += 1 + + record = MappingRecord( + variant_id=variant_id, + vrs_digest=(mv.pre_mapped or {}).get("id"), + pre_mapped=mv.pre_mapped or None, + assay_level=level, + hgvs_assay_level=hgvs_assay_level, + mapped_date=mv.mapped_date, + vrs_version=mv.vrs_version, + mapping_api_version=mv.mapping_api_version, + target_gene_mapping_id=mv.target_gene_mapping_id, + alignment_level=mv.alignment_level, + at_mismatched_locus=mv.at_mismatched_locus, + near_gap=mv.near_gap, + valid_from=valid_from, + valid_to=valid_to, + ) + db.add(record) + stats["records"] += 1 + + allele = _authoritative_allele(db, mv, level, allele_cache) + if allele is not None: + db.add( + MappingRecordAllele( + mapping_record=record, + allele=allele, + is_authoritative=True, + valid_from=valid_from, + valid_to=valid_to, + ) + ) + stats["record_alleles"] += 1 + + pending.append((mv, record, allele, hgvs_assay_level, valid_from, valid_to)) + + if not pending: + return set() + + db.flush() # one round trip resolves every record/allele/link id added for this whole variant + stats["variants"] += 1 + + touched_allele_ids: set[int] = set() + for mv, record, allele, hgvs_assay_level, valid_from, valid_to in pending: + # VRS_MAPPING: variant-subject, written for every version regardless of allele — mirrors the + # live job (mapping.py), whose FAILED/ABSENT outcomes get a record but no allele too. + if mv.error_message: + mapping_disposition = Disposition.FAILED + elif allele is not None: + mapping_disposition = Disposition.PRESENT + else: + mapping_disposition = Disposition.ABSENT + annotation_manager.record_event( + AnnotationType.VRS_MAPPING, + variant_id=variant_id, + disposition=mapping_disposition, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 1 + + if allele is not None: + # CLINGEN_ALLELE_ID / MAPPED_HGVS: allele-subject, only meaningful once an allele exists. + # allele.id is real now — this loop runs after the single flush above. + annotation_manager.record_event( + AnnotationType.CLINGEN_ALLELE_ID, + allele_id=allele.id, + disposition=Disposition.PRESENT if mv.clingen_allele_id else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + annotation_manager.record_event( + AnnotationType.MAPPED_HGVS, + allele_id=allele.id, + disposition=Disposition.PRESENT if hgvs_assay_level else Disposition.ABSENT, + reason=EventReason.MIGRATED.value, + source_version=mv.mapping_api_version, + ) + stats["annotation_events_written"] += 2 + + touched_allele_ids.add(allele.id) + + # Flushed once for the whole variant (not per event) — still inside this variant's own SAVEPOINT + # (the caller's `with db.begin_nested():` wraps this entire call), so a buffered event never + # outlives the rollback of the record/allele it references. See do_migration's except-block for the + # backstop: if something *between* here and there still raised, pending events for this variant are + # explicitly discarded rather than leaking into a later, unrelated flush. + annotation_manager.flush() + return touched_allele_ids + + +def _variant_ids_to_migrate(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Variant ids that have ``mapped_variants`` but no *live* ``MappingRecord`` yet. + + Keyed on a live record specifically, not "any record at all": a variant can have historical + records from a prior pass but still be missing its live one (the live version's mapped_variant had + no resolvable sequence level — see :func:`_migrate_variant`), and that variant must stay eligible + for retry until a re-map (or a QC fix) lets its live window resolve. The parallel-tables invariant + still makes "has mapped_variants and no live mapping record" the exact set of variants with + incomplete substrate; native new-model variants have no ``mapped_variants``. + """ + has_live_record = sa.select(MappingRecord.id).where( + MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None) + ) + query = ( + sa.select(Variant.id) + .join(MappedVariant, MappedVariant.variant_id == Variant.id) + .where(~has_live_record.exists()) + ) + if score_set_urn is not None: + query = query.join(ScoreSet, ScoreSet.id == Variant.score_set_id).where(ScoreSet.urn == score_set_urn) + return list(db.scalars(query.distinct().order_by(Variant.id)).all()) + + +_SKIP_REASONS = { + "no_current": "mapped_variants present but none current", + "no_level_historical": "a superseded mapped_variant with no resolvable sequence level (window skipped, siblings rescued)", + "no_level_live": "the LIVE mapped_variant has no resolvable sequence level — no live record could be created; remains eligible for retry", + "live_precedes_history": "the live mapped_variant's mapped_date precedes a historical one (data anomaly — a re-map candidate)", +} + + +def _log_skip_summary(db: Session, skipped: dict[str, list[int]], *, chunk: int = 5000) -> None: + """Roll skipped variants up into one log line per (reason, score set). + + Skips cluster hard by score set — a whole library the mapper couldn't convert skips en masse — so + a per-variant warning drowns the log while a per-score-set count pinpoints exactly which set to + chase. Variant ids are resolved to score set URNs in chunks to keep the ``IN`` clauses bounded. + """ + for reason, variant_ids in skipped.items(): + if not variant_ids: + continue + counts: Counter = Counter() + for offset in range(0, len(variant_ids), chunk): + part = variant_ids[offset : offset + chunk] + for urn, n in db.execute( + sa.select(ScoreSet.urn, sa.func.count()) + .join(Variant, Variant.score_set_id == ScoreSet.id) + .where(Variant.id.in_(part)) + .group_by(ScoreSet.urn) + ).all(): + counts[urn] += n + detail = _SKIP_REASONS.get(reason, reason) + logger.warning("Skipped %d variant(s) across %d score set(s) — %s:", len(variant_ids), len(counts), detail) + for urn, n in counts.most_common(): + logger.warning(" %s: %d variant(s) skipped (%s).", urn, n, reason) + + +def do_migration( + db: Session, + *, + score_set_urn: Optional[str] = None, + batch_size: int = 1000, + dry_run: bool = False, + run_at: Optional[datetime] = None, +) -> None: + """Backfill the allele substrate from ``mapped_variants``, committing per batch. + + ``run_at`` (default: now) is the single cutover instant used to force-close any protein-level + ClinVar link that would otherwise land live — see :func:`_write_allele_timeline`. Exposed as a + parameter mainly for deterministic tests; a real run just wants "now". + + Also backfills the ``annotation_event`` audit log — but only for the mapping/identity side + (``VRS_MAPPING``, ``CLINGEN_ALLELE_ID``, ``MAPPED_HGVS``), not gnomAD/ClinVar/VEP. That scope split + is deliberate: the mapping side is a genuine, non-lossy reconstruction (one real historical event + per ``mapped_variants`` version, the same data already trusted enough to build ``MappingRecord``/ + ``Allele`` from), whereas gnomAD/ClinVar/VEP would need to guess at a created/reconfirmed/skipped + cadence legacy data never recorded — and those sources get real, fresh events again once each score + set is re-annotated, so the audit gap there is temporary, not permanent. See + :data:`EventReason.MIGRATED`. + """ + print("Backfilling the Allele substrate from mapped_variants...") + if dry_run: + print(" DRY RUN — no changes will be committed.") + run_at = run_at or datetime.now(timezone.utc) + annotation_manager = AnnotationStatusManager(db) + + variant_ids = _variant_ids_to_migrate(db, score_set_urn) + print(f"Found {len(variant_ids)} variants to migrate.") + + stats: Counter = Counter() + # Wall-clock spent in each phase, so a run reports where the time goes (batch build vs. the + # final annotation pass) and whether the per-batch rate degrades as the tables grow. + phase_seconds: Counter = Counter() + # Ids only — a deduped allele can be shared by variants in different chunks, so its annotation + # timeline can only be assembled once every contributing variant has been migrated, but the actual + # gnomAD/ClinVar/VEP payload doesn't need to ride along in memory to get there: it's re-read fresh + # from the DB per allele-batch in the deferred pass below (_backfill_annotation_timelines). Holding + # just ids here (not full _AnnotationObservation objects) keeps this set's footprint tiny even at + # millions of variants — see that function's docstring for why the old accumulate-everything design + # degraded badly at scale. + touched_allele_ids: set[int] = set() + # Skipped variant ids by reason, rolled up into a per-score-set summary at the end instead of a + # per-variant warning (one unmappable library can skip hundreds of thousands of variants). + skipped: dict[str, list[int]] = defaultdict(list) + + run_start = perf_counter() + for offset in range(0, len(variant_ids), batch_size): + chunk = variant_ids[offset : offset + batch_size] + + mark = perf_counter() + mvs_by_variant: dict[int, list[MappedVariant]] = {vid: [] for vid in chunk} + for mv in db.scalars(sa.select(MappedVariant).where(MappedVariant.variant_id.in_(chunk))).all(): + mvs_by_variant[mv.variant_id].append(mv) + phase_seconds["fetch_mapped_variants"] += perf_counter() - mark + + # Bulk pre-fetch, once per chunk instead of once per window — this (not query execution time) + # was the actual bottleneck: a per-window round trip for "does this record already exist" plus + # another for "does this allele already exist" dominates wall-clock at millions-of-rows scale, + # far more than the cost of the queries themselves. + mark = perf_counter() + existing_records: set[tuple[int, date]] = set( + db.execute( + sa.select(MappingRecord.variant_id, MappingRecord.mapped_date).where( + MappingRecord.variant_id.in_(chunk) + ) + ).all() + ) + candidate_digests = { + (mv.post_mapped or {}).get("id") for mvs in mvs_by_variant.values() for mv in mvs + } + candidate_digests.discard(None) + allele_cache: dict[str, Allele] = {} + if candidate_digests: + for allele in db.scalars(sa.select(Allele).where(Allele.vrs_digest.in_(candidate_digests))).all(): + allele_cache[allele.vrs_digest] = allele + phase_seconds["prefetch"] += perf_counter() - mark + + mark = perf_counter() + for variant_id in chunk: + # A per-variant SAVEPOINT so one bad variant rolls back only its own inserts (not the + # whole batch), and its touched allele ids are simply never added rather than referencing + # rolled-back rows. + pending_events_before = len(annotation_manager._pending) + try: + with db.begin_nested(): + variant_touched_allele_ids = _migrate_variant( + db, + variant_id, + mvs_by_variant[variant_id], + stats, + skipped, + annotation_manager, + existing_records, + allele_cache, + ) + except Exception as exc: # noqa: BLE001 — one bad variant must not abort the batch + # Discard any events this variant buffered but never got to flush before raising — they + # would otherwise linger in the manager's queue and get inserted by a later, unrelated + # flush, still referencing this variant/allele's now-rolled-back id (an FK violation far + # from its actual cause). _migrate_variant flushes its own events before returning + # normally, so this is a no-op on the success path. + del annotation_manager._pending[pending_events_before:] + stats["variants_errored"] += 1 + logger.exception("Failed to migrate variant %s: %s", variant_id, exc) + continue + touched_allele_ids.update(variant_touched_allele_ids) + phase_seconds["build_records"] += perf_counter() - mark + + mark = perf_counter() + if dry_run: + db.rollback() + touched_allele_ids.clear() # nothing persisted this batch; these ids are now invalid + else: + db.commit() + phase_seconds["commit"] += perf_counter() - mark + + processed = min(offset + batch_size, len(variant_ids)) + elapsed = perf_counter() - run_start + rate = processed / elapsed if elapsed else 0.0 + print(f" Processed {processed}/{len(variant_ids)} variants... ({rate:.0f}/s, {elapsed:.0f}s elapsed)") + + if not dry_run and touched_allele_ids: + print(f"Building annotation timelines for {len(touched_allele_ids)} allele(s)...") + mark = perf_counter() + _backfill_annotation_timelines( + db, sorted(touched_allele_ids), batch_size=batch_size, run_at=run_at, stats=stats + ) + phase_seconds["annotation_timelines"] += perf_counter() - mark + + print("\nMigration completed:") + for key in ( + "variants", + "records", + "record_alleles", + "hgvs_recovered_from_level_column", + "hgvs_recovered_from_vrs_expression", + "annotation_events_written", + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + "variants_skipped_no_current", + "windows_skipped_no_level_historical", + "windows_skipped_no_level_live", + "variants_live_precedes_history", + "variants_errored", + ): + print(f" {key}: {stats[key]}") + + _log_skip_summary(db, skipped) + + total = perf_counter() - run_start + print("\nTiming by phase:") + for phase in ("fetch_mapped_variants", "build_records", "commit", "annotation_timelines"): + seconds = phase_seconds[phase] + share = f"{100 * seconds / total:.0f}%" if total else "n/a" + print(f" {phase}: {seconds:.1f}s ({share})") + variants_done = stats["variants"] + overall_rate = f"{variants_done / total:.0f}/s" if total else "n/a" + print(f" total: {total:.1f}s ({overall_rate})") + + +def _allele_ids_to_rebuild(db: Session, score_set_urn: Optional[str]) -> list[int]: + """Every allele id with an authoritative link, optionally scoped to one score set's variants.""" + query = sa.select(sa.distinct(MappingRecordAllele.allele_id)).where(MappingRecordAllele.is_authoritative) + if score_set_urn is not None: + query = ( + query.join(MappingRecord, MappingRecordAllele.mapping_record_id == MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .join(ScoreSet, ScoreSet.id == Variant.score_set_id) + .where(ScoreSet.urn == score_set_urn) + ) + return list(db.scalars(query.order_by(MappingRecordAllele.allele_id)).all()) + + +def rebuild_annotation_timelines( + db: Session, *, score_set_urn: Optional[str] = None, batch_size: int = 1000, run_at: Optional[datetime] = None +) -> None: + """(Re)build annotation link timelines for already-migrated variants, independently of record + creation. Idempotent (existence-checked, savepoint-isolated). Use to recover after a crash in the + ``migrate`` annotation pass, or to refresh timelines after fixing annotation logic. + + Rebuilds strictly from the legacy proxy data (``mapped_variants`` associations), same as + ``migrate`` — it is not the cross-level fan-out job — so it applies the same protein-level ClinVar + closure rule (see :func:`_write_allele_timeline`) using its own ``run_at`` (default: now). Batches by + allele id via :func:`_backfill_annotation_timelines`, the same bounded-memory pass ``migrate`` itself + uses — this used to accumulate every observation for the whole rebuild in memory before writing + anything, which had the identical unbounded-growth problem the ``migrate`` pass was fixed for. + """ + print("Rebuilding annotation timelines from existing records...") + run_at = run_at or datetime.now(timezone.utc) + allele_ids = _allele_ids_to_rebuild(db, score_set_urn) + print(f"Found {len(allele_ids)} allele(s) with authoritative links.") + + stats: Counter = Counter() + _backfill_annotation_timelines(db, allele_ids, batch_size=batch_size, run_at=run_at, stats=stats) + + print("\nRebuild completed:") + for key in ( + "gnomad_links", + "protein_gnomad_links_closed", + "clinvar_links", + "protein_clinvar_links_closed", + "vep_consequences", + "vep_source_version_resolved", + "vep_source_version_legacy_fallback", + "annotation_conflicts", + ): + print(f" {key}: {stats[key]}") + + +def verify_migration(db: Session, *, score_set_urn: Optional[str] = None) -> None: + """Report substrate coverage without writing.""" + print("\nVerifying backfill...") + + remaining = len(_variant_ids_to_migrate(db, score_set_urn)) + total_mapped_variant_variants = db.scalar(sa.select(sa.func.count(sa.distinct(MappedVariant.variant_id)))) + total_records = db.scalar(sa.select(sa.func.count(MappingRecord.id))) + live_records = db.scalar(sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.valid_to.is_(None))) + total_alleles = db.scalar(sa.select(sa.func.count(Allele.id))) + live_auth_links = db.scalar( + sa.select(sa.func.count(MappingRecordAllele.id)).where( + MappingRecordAllele.is_authoritative, MappingRecordAllele.valid_to.is_(None) + ) + ) + gnomad_links = db.scalar(sa.select(sa.func.count(GnomadAlleleLink.id))) + clinvar_links = db.scalar(sa.select(sa.func.count(ClinvarAlleleLink.id))) + vep_consequences = db.scalar(sa.select(sa.func.count(VepAlleleConsequence.id))) + migrated_events = db.scalar( + sa.select(sa.func.count(AnnotationEvent.id)).where(AnnotationEvent.reason == EventReason.MIGRATED.value) + ) + + print(f" Variants with mapped_variants: {total_mapped_variant_variants}") + print(f" Variants still needing backfill: {remaining}") + print(f" MappingRecords (total / live): {total_records} / {live_records}") + print(f" Alleles (deduped): {total_alleles}") + print(f" Live authoritative links: {live_auth_links}") + print(f" gnomAD / ClinVar / VEP annotations: {gnomad_links} / {clinvar_links} / {vep_consequences}") + print(f" MIGRATED annotation_event rows: {migrated_events}") + if remaining == 0: + print(" ✓ Every legacy variant has a MappingRecord.") + else: + print(f" ⚠ {remaining} variants remain (see logged anomalies, or re-run migrate).") + + +def rollback_migration(db: Session) -> None: + """Delete backfilled ``MappingRecord``s (cascading their allele links). + + Scoped to variants that also have ``mapped_variants`` — the exact discriminator + for backfilled records, since native new-model variants have no legacy rows. + Deduped ``Allele``s and annotation links are left in place: alleles are content + -addressed and harmless if orphaned (a re-run reuses them), and annotation + links re-existence-check. Delete those manually if a full teardown is needed. + + ``AnnotationEvent`` rows (reason ``MIGRATED``) are left in place too, on purpose: it is an + append-only audit log, and deleting them on rollback would erase the very audit trail it exists to + keep — a re-run after rollback just appends a fresh set (matching how ``MappingRecord`` itself gets + deleted and recreated), not a duplicate-avoidance concern the way the ValidTime domain tables are. + """ + print("Rolling back backfilled MappingRecords (scoped to variants with mapped_variants)...") + legacy_variant_ids = sa.select(sa.distinct(MappedVariant.variant_id)) + count = db.scalar( + sa.select(sa.func.count(MappingRecord.id)).where(MappingRecord.variant_id.in_(legacy_variant_ids)) + ) + # ON DELETE CASCADE on mapping_record_alleles.mapping_record_id removes the links. + db.execute(sa.delete(MappingRecord).where(MappingRecord.variant_id.in_(legacy_variant_ids))) + db.commit() + print(f"Deleted {count} MappingRecords (and their allele links via cascade).") + print("Note: deduped Alleles, annotation links, and MIGRATED annotation_event rows were left in place") + print("(re-runnable / harmless orphans; the event log is append-only by design).") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Backfill the Allele substrate from mapped_variants.") + parser.add_argument( + "command", nargs="?", default="migrate", choices=["migrate", "verify", "rollback", "rebuild-annotations"] + ) + parser.add_argument("--score-set", dest="score_set_urn", default=None, help="Limit to one score set URN.") + parser.add_argument("--batch-size", type=int, default=1000, help="Variants per commit (default 1000).") + parser.add_argument("--dry-run", action="store_true", help="Run without committing (rolls back each batch).") + return parser.parse_args() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + args = _parse_args() + + if args.command == "rollback": + print("WARNING: this deletes all backfilled MappingRecords and their allele links.") + if input("Are you sure you want to continue? (y/N): ").lower() == "y": + with SessionLocal() as session: + rollback_migration(session) + else: + print("Rollback cancelled.") + elif args.command == "verify": + with SessionLocal() as session: + verify_migration(session, score_set_urn=args.score_set_urn) + elif args.command == "rebuild-annotations": + with SessionLocal() as session: + rebuild_annotation_timelines(session, score_set_urn=args.score_set_urn, batch_size=args.batch_size) + verify_migration(session, score_set_urn=args.score_set_urn) + else: + with SessionLocal() as session: + do_migration( + session, + score_set_urn=args.score_set_urn, + batch_size=args.batch_size, + dry_run=args.dry_run, + ) + verify_migration(session, score_set_urn=args.score_set_urn) diff --git a/src/mavedb/models/enums/event_reason.py b/src/mavedb/models/enums/event_reason.py index 35aa23af4..2082a6163 100644 --- a/src/mavedb/models/enums/event_reason.py +++ b/src/mavedb/models/enums/event_reason.py @@ -10,8 +10,17 @@ class EventReason(str, Enum): contributes its own pre-existing domain enum instead of duplicating here: ``mapping`` uses ``MappingOutcome`` (mapped / intronic / no_protein_consequence / failed), which mirrors the external dcd-mapping vocabulary. The full ``reason`` vocabulary is this enum plus that one. + + ``MIGRATED`` is the one exception to "reason means what happened": it deliberately does not + distinguish created/preexisting/reconfirmed etc., because a one-off data migration reconstructing + state from a legacy source cannot know which of those actually happened historically — only + ``disposition`` (present/absent/failed) is knowable there. Using it elsewhere would misrepresent a + live job's real observation as a backfill guess. """ + # spans present / absent / failed — a reconstructed observation, not a live run + MIGRATED = "migrated" # state reconstructed from a world lacking now required context; no per-run history + # present — we hold the result / the step succeeded CREATED = "created" # linked/registered this run (gnomAD, ClinVar, CAR) PREEXISTING = "preexisting" # already held before this run (gnomAD, ClinVar, CAR) From b3fd1ba21a0a1d044f5c2d06f3ed5aea97154111 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 20 Jul 2026 17:14:49 -0700 Subject: [PATCH 86/93] feat(serving): rewrite annotation views and stats MV onto mapping_records The read layer (v_variant_annotations, published_variants MV, statistics router) still joined the legacy mapped_variants table, so it never resolved the mapping_records/alleles substrate the mapping job now writes to. This completes the read-side cutover onto the new tables. - rewrite v_variant_annotations to pivot the record's live allele links into the flat hgvs_g/hgvs_c/hgvs_p triple, and pull clingen_allele_id and VEP consequence from the authoritative allele, via correlated scalar subqueries - rebuild published_variants_materialized_view on mapping_record_id / current_mapping_record, keeping every record version (not just the live one) so history-aware counts still work - update statistics router and tests to the renamed columns - freeze all historical migrations' view/MV SQL as inline literals instead of importing from the model, so replaying an old revision builds the shape that existed at that point in history, not whatever the model looks like after later migrations rewrite it - add view-level tests for the flat-triple reconstruction, protein-only assays, and unmapped variants under the new substrate --- ..._simplify_published_variants_mv_indexes.py | 7 +- ...rite_v_variant_annotations_onto_alleles.py | 91 ++++++++++++++ ...uild_published_variants_mv_onto_records.py | 86 +++++++++++++ ...aterialized_view_for_variant_statistics.py | 59 +++++---- ...b3c4d5e6_add_v_variant_annotations_view.py | 33 ++++- src/mavedb/models/published_variant.py | 22 ++-- src/mavedb/models/variant_annotation_view.py | 111 ++++++++++++++--- src/mavedb/routers/statistics.py | 10 +- tests/conftest.py | 4 +- tests/models/test_variant_annotation_view.py | 115 ++++++++++++++++++ tests/routers/test_statistics.py | 12 +- 11 files changed, 485 insertions(+), 65 deletions(-) create mode 100644 alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py create mode 100644 alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py create mode 100644 tests/models/test_variant_annotation_view.py diff --git a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py index 479c51a33..1ce888933 100644 --- a/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py +++ b/alembic/versions/4726e4dddde8_simplify_published_variants_mv_indexes.py @@ -8,8 +8,6 @@ from alembic import op -from mavedb.models.published_variant import signature - # revision identifiers, used by Alembic. revision = "4726e4dddde8" @@ -17,6 +15,11 @@ branch_labels = None depends_on = None +# Frozen literal, not imported from the model: this migration replaces the historical +# mapped_variants-based MV indexes at its point in history. A later migration rebuilds the MV onto +# the mapping_records substrate and re-indexes it, so the column names below stay the legacy ones. +signature = "published_variants_materialized_view" + def upgrade(): # ### commands auto generated by Alembic - please adjust! ### diff --git a/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py new file mode 100644 index 000000000..c8f702d72 --- /dev/null +++ b/alembic/versions/a1d4f7c2e9b0_rewrite_v_variant_annotations_onto_alleles.py @@ -0,0 +1,91 @@ +"""rewrite v_variant_annotations onto the mapping_records/alleles substrate + +Revision ID: a1d4f7c2e9b0 +Revises: e7b2c9a1f4d3 +Create Date: 2026-07-18 + +Drops the legacy ``mapped_variants``-based ``v_variant_annotations`` view and recreates it over the +``MappingRecord`` / ``Allele`` / ``MappingRecordAllele`` substrate (part of the MappedVariant +read-path teardown). Column changes vs. the legacy shape: + * ``mapped_variant_id`` -> ``mapping_record_id`` (the mapping identity is now the mapping record). + * ``mapping_error`` dropped — there is no per-mapping-record error column; per-subject annotation + dispositions live in ``v_current_annotation_events`` and score-set-level mapping errors on + ``scoresets.mapping_errors``. + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``variant_annotation_view.definition`` would +couple this fixed revision to mutable code, so the next model edit (the following view rewrite) would +make a replay of *this* revision build that later shape — against tables that may not exist yet at this +point in history — and fail. ``_UPGRADE_DEFINITION`` below was compiled once from the model at +authoring time (``CreateView(signature, definition).compile(postgresql.dialect())``) and must not be +edited to track the model; the next shape change gets its own migration. +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a1d4f7c2e9b0" +down_revision = "e7b2c9a1f4d3" +branch_labels = None +depends_on = None + +SIGNATURE = "v_variant_annotations" + +# New (mapping_records/alleles) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. Allele-derived columns are correlated scalar +# subqueries so the grain stays one row per (variant, annotation_type), matching the legacy view. +_UPGRADE_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapping_records.id AS mapping_record_id, (SELECT alleles.clingen_allele_id +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS clingen_allele_id, mapping_records.hgvs_assay_level, (SELECT alleles.hgvs_g +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'genomic' + LIMIT 1) AS hgvs_g, (SELECT alleles.hgvs_c +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'cdna' + LIMIT 1) AS hgvs_c, (SELECT alleles.hgvs_p +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.valid_to IS NULL AND alleles.level = 'protein' + LIMIT 1) AS hgvs_p, (SELECT vep_allele_consequences.functional_consequence +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_functional_consequence, (SELECT vep_allele_consequences.access_date +FROM mapping_record_alleles JOIN alleles ON alleles.id = mapping_record_alleles.allele_id JOIN vep_allele_consequences ON vep_allele_consequences.allele_id = alleles.id AND vep_allele_consequences.valid_to IS NULL +WHERE mapping_record_alleles.mapping_record_id = mapping_records.id AND mapping_record_alleles.is_authoritative IS true AND mapping_record_alleles.valid_to IS NULL + LIMIT 1) AS vep_access_date, mapping_records.mapped_date, mapping_records.mapping_api_version, mapping_records.vrs_version, variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapping_records ON mapping_records.variant_id = variants.id AND mapping_records.valid_to IS NULL + LEFT OUTER JOIN variant_annotation_status ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + +# Legacy (mapped_variants-based) definition, inlined so the downgrade can restore the prior shape. +# ``mapped_variants`` still exists at this revision (dropped in a later teardown migration), so this +# remains runnable on downgrade. +_LEGACY_DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + + +def upgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_UPGRADE_DEFINITION}") + + +def downgrade() -> None: + op.execute(f"DROP VIEW {SIGNATURE}") + op.execute(f"CREATE VIEW {SIGNATURE} AS {_LEGACY_DEFINITION}") diff --git a/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py new file mode 100644 index 000000000..2f9dad721 --- /dev/null +++ b/alembic/versions/b2e5a8d3f0c1_rebuild_published_variants_mv_onto_records.py @@ -0,0 +1,86 @@ +"""rebuild published_variants MV onto the mapping_records substrate + +Revision ID: b2e5a8d3f0c1 +Revises: a1d4f7c2e9b0 +Create Date: 2026-07-18 + +Rebuilds the ``published_variants_materialized_view`` off the legacy ``mapped_variants`` join onto the +``mapping_records`` substrate (part of the MappedVariant read-path teardown). The MV keeps its grain +(one row per published variant per mapping-record version, unmapped variants retained with a NULL +mapping id) and its published filter. Column changes: + * ``mapped_variant_id`` -> ``mapping_record_id`` + * ``current_mapped_variant`` -> ``current_mapping_record`` (``valid_to IS NULL``) + +Both directions are frozen inline as literal SQL — the migration imports nothing from the model layer. +A migration is an immutable historical record: importing ``published_variant.definition`` would couple +this fixed revision to mutable code, so a later model edit would make a replay of *this* revision build +that later shape (possibly against tables that no longer exist by then) and fail. ``_UPGRADE_DEFINITION`` +below was compiled once from the model at authoring time and must not be edited to track the model. +""" + +from alembic_utils.pg_materialized_view import PGMaterializedView + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "b2e5a8d3f0c1" +down_revision = "a1d4f7c2e9b0" +branch_labels = None +depends_on = None + +SIGNATURE = "published_variants_materialized_view" + +# New (mapping_records-based) definition, frozen at this revision. Compiled from the model once at +# authoring time; do not edit to track the model. The LEFT OUTER JOIN to mapping_records carries no +# liveness filter on purpose — every record version is retained and current-ness is exposed via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). +_UPGRADE_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapping_records.id AS mapping_record_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapping_records.valid_to IS NULL AS current_mapping_record +FROM variants LEFT OUTER JOIN mapping_records ON variants.id = mapping_records.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + +# Legacy (mapped_variants-based) definition, inlined for the downgrade path. ``mapped_variants`` still +# exists at this revision (dropped in a later teardown migration), so downgrade remains runnable. +_LEGACY_DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + + +def upgrade(): + # Dropping the MV cascades to its owned indexes (the legacy idx_..._ids on mapped_variant_id). + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapping_record_id", "variant_id", "score_set_id"], + unique=True, + ) + + +def downgrade(): + op.drop_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_UPGRADE_DEFINITION, with_data=True) + ) + op.create_entity( + PGMaterializedView(schema="public", signature=SIGNATURE, definition=_LEGACY_DEFINITION, with_data=True) + ) + op.create_index( + f"idx_{SIGNATURE}_ids", + SIGNATURE, + ["mapped_variant_id", "variant_id", "score_set_id"], + unique=True, + ) diff --git a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py index 7ce51c88d..3b3855e7a 100644 --- a/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py +++ b/alembic/versions/b85bc7b1bec7_materialized_view_for_variant_statistics.py @@ -2,13 +2,16 @@ Revision ID: b85bc7b1bec7 Revises: c404b6719110 Create Date: 2025-03-14 01:53:19.898198 + +The MV SQL is inlined as a frozen literal rather than imported from the model. At this revision the +MV joins the (now legacy) ``mapped_variants`` table; a later migration rebuilds it onto the +``mapping_records`` substrate. Importing the live model definition here would make a fresh replay +build *that* later shape at this early revision — before the substrate tables exist — and fail. +Freezing the historical SQL keeps replay green; the rebuild happens in its own migration. """ from alembic import op from alembic_utils.pg_materialized_view import PGMaterializedView -from sqlalchemy.dialects import postgresql - -from mavedb.models.published_variant import signature, definition # revision identifiers, used by Alembic. @@ -17,59 +20,71 @@ branch_labels = None depends_on = None +SIGNATURE = "published_variants_materialized_view" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.id AS variant_id, variants.urn AS variant_urn, mapped_variants.id AS mapped_variant_id, + scoresets.id AS score_set_id, scoresets.urn AS score_set_urn, scoresets.published_date AS published_date, + mapped_variants.current AS current_mapped_variant +FROM variants LEFT OUTER JOIN mapped_variants ON variants.id = mapped_variants.variant_id + JOIN scoresets ON scoresets.id = variants.scoreset_id +WHERE scoresets.published_date IS NOT NULL +""" + def upgrade(): op.create_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) op.create_index( - f"idx_{signature}_variant_id", - signature, + f"idx_{SIGNATURE}_variant_id", + SIGNATURE, ["variant_id"], unique=False, ) op.create_index( - f"idx_{signature}_variant_urn", - signature, + f"idx_{SIGNATURE}_variant_urn", + SIGNATURE, ["variant_urn"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_id", - signature, + f"idx_{SIGNATURE}_score_set_id", + SIGNATURE, ["score_set_id"], unique=False, ) op.create_index( - f"idx_{signature}_score_set_urn", - signature, + f"idx_{SIGNATURE}_score_set_urn", + SIGNATURE, ["score_set_urn"], unique=False, ) op.create_index( - f"idx_{signature}_mapped_variant_id", - signature, + f"idx_{SIGNATURE}_mapped_variant_id", + SIGNATURE, ["mapped_variant_id"], unique=True, ) def downgrade(): - op.drop_index(f"idx_{signature}_variant_id", signature) - op.drop_index(f"idx_{signature}_variant_urn", signature) - op.drop_index(f"idx_{signature}_mapped_variant_id", signature) - op.drop_index(f"idx_{signature}_score_set_id", signature) - op.drop_index(f"idx_{signature}_score_set_urn", signature) + op.drop_index(f"idx_{SIGNATURE}_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_variant_urn", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_mapped_variant_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_id", SIGNATURE) + op.drop_index(f"idx_{SIGNATURE}_score_set_urn", SIGNATURE) op.drop_entity( PGMaterializedView( schema="public", - signature=signature, - definition=definition.compile(dialect=postgresql.dialect()).string, + signature=SIGNATURE, + definition=DEFINITION, with_data=True, ) ) diff --git a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py index 12ffd574d..2d377def7 100644 --- a/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py +++ b/alembic/versions/f2a1b3c4d5e6_add_v_variant_annotations_view.py @@ -7,23 +7,46 @@ Creates the v_variant_annotations convenience view, which joins variants, mapped_variants, variant_annotation_status, and scoresets into a single flat row per (variant, annotation_type). Intended for operator queries and the variant_annotations CLI script. + +The view SQL is inlined as a frozen literal rather than imported from the model. At this point in +history the view joins the (now legacy) ``mapped_variants`` table; a later migration rewrites it onto +the ``mapping_records`` / ``alleles`` substrate. Importing the live model definition here would make a +fresh replay build *that* later shape at this early revision — before the substrate tables exist — +and fail. Freezing the historical SQL keeps replay green; the rewrite happens in its own migration. """ from alembic import op -from mavedb.db.view import CreateView, DropView -from mavedb.models.variant_annotation_view import definition, signature - # revision identifiers, used by Alembic. revision = "f2a1b3c4d5e6" down_revision = "e3a7b9f1d2c5" branch_labels = None depends_on = None +SIGNATURE = "v_variant_annotations" + +# Historical definition, frozen at this revision. Do not edit to track the model. +DEFINITION = """ +SELECT variants.urn AS variant_urn, scoresets.urn AS score_set_urn, variants.hgvs_nt, variants.hgvs_pro, + variants.hgvs_splice, mapped_variants.id AS mapped_variant_id, mapped_variants.clingen_allele_id, + mapped_variants.hgvs_assay_level, mapped_variants.hgvs_g, mapped_variants.hgvs_c, mapped_variants.hgvs_p, + mapped_variants.vep_functional_consequence, mapped_variants.vep_access_date, mapped_variants.mapped_date, + mapped_variants.mapping_api_version, mapped_variants.vrs_version, mapped_variants.error_message AS mapping_error, + variant_annotation_status.annotation_type, variant_annotation_status.status AS annotation_status, + variant_annotation_status.failure_category, variant_annotation_status.error_message AS annotation_error, + variant_annotation_status.version AS annotation_version, variant_annotation_status.annotation_metadata, + variant_annotation_status.current, variant_annotation_status.created_at AS annotation_created_at, + variant_annotation_status.updated_at AS annotation_updated_at +FROM variants JOIN scoresets ON scoresets.id = variants.scoreset_id + LEFT OUTER JOIN mapped_variants ON mapped_variants.variant_id = variants.id AND mapped_variants.current = true + LEFT OUTER JOIN variant_annotation_status + ON variant_annotation_status.variant_id = variants.id AND variant_annotation_status.current = true +""" + def upgrade() -> None: - op.execute(CreateView(signature, definition, materialized=False)) + op.execute(f"CREATE VIEW {SIGNATURE} AS {DEFINITION}") def downgrade() -> None: - op.execute(DropView(signature, materialized=False)) + op.execute(f"DROP VIEW {SIGNATURE}") diff --git a/src/mavedb/models/published_variant.py b/src/mavedb/models/published_variant.py index eff749bd7..503d3fa30 100644 --- a/src/mavedb/models/published_variant.py +++ b/src/mavedb/models/published_variant.py @@ -1,25 +1,29 @@ -from sqlalchemy import select, join +from sqlalchemy import join, select from mavedb.db.view import MaterializedView, view - +from mavedb.models.mapping_record import MappingRecord from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant -from mavedb.models.mapped_variant import MappedVariant - signature = "published_variants_materialized_view" + +# One row per (published variant, mapping-record version). The LEFT OUTER JOIN to mapping_records +# carries NO liveness filter on purpose: every record version (live and superseded) is retained so +# the statistics layer can count either the current mappings or the full history via the +# ``current_mapping_record`` flag (``valid_to IS NULL``). Unmapped variants stay in the view with a +# NULL ``mapping_record_id``. definition = ( select( Variant.id.label("variant_id"), Variant.urn.label("variant_urn"), - MappedVariant.id.label("mapped_variant_id"), + MappingRecord.id.label("mapping_record_id"), ScoreSet.id.label("score_set_id"), ScoreSet.urn.label("score_set_urn"), ScoreSet.published_date.label("published_date"), - MappedVariant.current.label("current_mapped_variant"), + MappingRecord.valid_to.is_(None).label("current_mapping_record"), ) .select_from( - join(Variant, MappedVariant, Variant.id == MappedVariant.variant_id, isouter=True).join( + join(Variant, MappingRecord, Variant.id == MappingRecord.variant_id, isouter=True).join( ScoreSet, ScoreSet.id == Variant.score_set_id ) ) @@ -38,8 +42,8 @@ class PublishedVariantsMV(MaterializedView): variant_id = __table__.c.variant_id variant_urn = __table__.c.variant_urn - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id score_set_id = __table__.c.score_set_id score_set_urn = __table__.c.score_set_urn published_date = __table__.c.published_date - current_mapped_variant = __table__.c.current_mapped_variant + current_mapping_record = __table__.c.current_mapping_record diff --git a/src/mavedb/models/variant_annotation_view.py b/src/mavedb/models/variant_annotation_view.py index c7c77d6f8..6b6e52963 100644 --- a/src/mavedb/models/variant_annotation_view.py +++ b/src/mavedb/models/variant_annotation_view.py @@ -2,13 +2,89 @@ from mavedb.db.base import Base from mavedb.db.view import view -from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.allele import Allele +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant from mavedb.models.variant_annotation_status import VariantAnnotationStatus +from mavedb.models.vep_allele_consequence import VepAlleleConsequence signature = "v_variant_annotations" +# All allele-derived columns are correlated scalar subqueries keyed on the outer MappingRecord, so the +# outer query's FROM stays Variant / ScoreSet / MappingRecord / VariantAnnotationStatus and never +# references Allele / MappingRecordAllele directly. That keeps the grain at one row per +# (variant, annotation_type) — matching the legacy view — and, crucially, avoids ``aliased()`` at +# module import (which would force full mapper configuration before every model is registered, breaking +# a bare ``import`` of this module by Alembic). + + +def _auth_allele(column): + """Scalar subquery: ``column`` of the outer record's live **authoritative** (measured) allele. + + Source of the single-valued ``clingen_allele_id``. The authoritative link is 1:1 with the record + (the mapping job upholds one live authoritative link per record), so this resolves a single allele. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _auth_vep(column): + """Scalar subquery: ``column`` of the live VEP consequence on the record's authoritative allele.""" + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .join( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.valid_to.is_(None)), + ) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.valid_to.is_(None), + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +def _record_hgvs(level: str, column): + """Scalar subquery: the canonical HGVS at ``level`` for the outer record. + + Each mapped level (genomic / cdna / protein) is a distinct deduplicated :class:`Allele` linked to + the record. This constructs a flat ``hgvs_g`` / ``hgvs_c`` / ``hgvs_p`` triple by pivoting the record's + live allele links by level — yielding ``NULL`` where the record has no allele at that level. + """ + return ( + select(column) + .select_from(MappingRecordAllele) + .join(Allele, Allele.id == MappingRecordAllele.allele_id) + .where( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.valid_to.is_(None), + Allele.level == level, + ) + .limit(1) + .correlate(MappingRecord) + .scalar_subquery() + ) + + +# Flat operator/CLI convenience view: one row per (variant, current annotation_type). definition = ( select( Variant.urn.label("variant_urn"), @@ -16,18 +92,17 @@ Variant.hgvs_nt, Variant.hgvs_pro, Variant.hgvs_splice, - MappedVariant.id.label("mapped_variant_id"), - MappedVariant.clingen_allele_id, - MappedVariant.hgvs_assay_level, - MappedVariant.hgvs_g, - MappedVariant.hgvs_c, - MappedVariant.hgvs_p, - MappedVariant.vep_functional_consequence, - MappedVariant.vep_access_date, - MappedVariant.mapped_date, - MappedVariant.mapping_api_version, - MappedVariant.vrs_version, - MappedVariant.error_message.label("mapping_error"), + MappingRecord.id.label("mapping_record_id"), + _auth_allele(Allele.clingen_allele_id).label("clingen_allele_id"), + MappingRecord.hgvs_assay_level, + _record_hgvs("genomic", Allele.hgvs_g).label("hgvs_g"), + _record_hgvs("cdna", Allele.hgvs_c).label("hgvs_c"), + _record_hgvs("protein", Allele.hgvs_p).label("hgvs_p"), + _auth_vep(VepAlleleConsequence.functional_consequence).label("vep_functional_consequence"), + _auth_vep(VepAlleleConsequence.access_date).label("vep_access_date"), + MappingRecord.mapped_date, + MappingRecord.mapping_api_version, + MappingRecord.vrs_version, VariantAnnotationStatus.annotation_type, VariantAnnotationStatus.status.label("annotation_status"), VariantAnnotationStatus.failure_category, @@ -40,10 +115,9 @@ ) .select_from(Variant) .join(ScoreSet, ScoreSet.id == Variant.score_set_id) - .outerjoin( - MappedVariant, - and_(MappedVariant.variant_id == Variant.id, MappedVariant.current == True), # noqa: E712 - ) + # The variant's live mapping record. live-ness rides the ON clause (never a WHERE) so the outer + # join is preserved for unmapped variants — a WHERE would reject the null-record row. + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.valid_to.is_(None))) .outerjoin( VariantAnnotationStatus, and_(VariantAnnotationStatus.variant_id == Variant.id, VariantAnnotationStatus.current == True), # noqa: E712 @@ -59,7 +133,7 @@ class VariantAnnotationView(Base): hgvs_nt = __table__.c.hgvs_nt hgvs_pro = __table__.c.hgvs_pro hgvs_splice = __table__.c.hgvs_splice - mapped_variant_id = __table__.c.mapped_variant_id + mapping_record_id = __table__.c.mapping_record_id clingen_allele_id = __table__.c.clingen_allele_id hgvs_assay_level = __table__.c.hgvs_assay_level hgvs_g = __table__.c.hgvs_g @@ -70,7 +144,6 @@ class VariantAnnotationView(Base): mapped_date = __table__.c.mapped_date mapping_api_version = __table__.c.mapping_api_version vrs_version = __table__.c.vrs_version - mapping_error = __table__.c.mapping_error annotation_type = __table__.c.annotation_type annotation_status = __table__.c.annotation_status failure_category = __table__.c.failure_category diff --git a/src/mavedb/routers/statistics.py b/src/mavedb/routers/statistics.py index e52d2582d..5deea5636 100644 --- a/src/mavedb/routers/statistics.py +++ b/src/mavedb/routers/statistics.py @@ -345,7 +345,7 @@ def record_mapped_variant_counts(db: Session = Depends(get_db)) -> dict[str, int within a given record. """ variants = db.execute( - select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapped_variant_id)) + select(PublishedVariantsMV.score_set_urn, func.count(PublishedVariantsMV.mapping_record_id)) .group_by(PublishedVariantsMV.score_set_urn) .order_by(PublishedVariantsMV.score_set_urn) ).all() @@ -650,10 +650,10 @@ def mapped_variant_counts( """ # Fast path: total distinct mapped variants (optionally only current) without per-date aggregation. if group is None: - total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapped_variant_id))) + total_stmt = select(func.count(func.distinct(PublishedVariantsMV.mapping_record_id))) if onlyCurrent: - total_stmt = total_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + total_stmt = total_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) total = db.execute(total_stmt).scalar_one() # type: ignore return OrderedDict([("count", total)]) @@ -661,11 +661,11 @@ def mapped_variant_counts( # Grouped path: materialize distinct counts per published_date, then roll up. per_date_stmt = select( PublishedVariantsMV.published_date, - func.count(func.distinct(PublishedVariantsMV.mapped_variant_id)), + func.count(func.distinct(PublishedVariantsMV.mapping_record_id)), ) if onlyCurrent: - per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapped_variant.is_(True)) + per_date_stmt = per_date_stmt.where(PublishedVariantsMV.current_mapping_record.is_(True)) per_date = db.execute( per_date_stmt.group_by(PublishedVariantsMV.published_date).order_by(PublishedVariantsMV.published_date) diff --git a/tests/conftest.py b/tests/conftest.py index 34e366392..d2825fcdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,11 +95,11 @@ def session(postgresql): Base.metadata.create_all(bind=engine) # Create a unique index for the published_variants_materialized_view to - # enforce uniqueness on (variant_id, mapped_variant_id, score_set_id). This + # enforce uniqueness on (variant_id, mapping_record_id, score_set_id). This # allows us to test mat view refreshes that require this constraint. session.execute( text("""CREATE UNIQUE INDEX IF NOT EXISTS published_variants_mv_unique_idx - ON published_variants_materialized_view (variant_id, mapped_variant_id, score_set_id)"""), + ON published_variants_materialized_view (variant_id, mapping_record_id, score_set_id)"""), ) session.commit() diff --git a/tests/models/test_variant_annotation_view.py b/tests/models/test_variant_annotation_view.py new file mode 100644 index 000000000..d1c2f1d72 --- /dev/null +++ b/tests/models/test_variant_annotation_view.py @@ -0,0 +1,115 @@ +# ruff: noqa: E402 +"""Tests for the v_variant_annotations view after its rewrite onto the MappingRecord/Allele substrate. + +The view is a flat operator convenience projection: one row per (variant, current annotation_type). It +reconstructs the legacy ``mapped_variants`` row's flat ``hgvs_g``/``hgvs_c``/``hgvs_p`` triple by +pivoting the record's live allele links by level, and carries the single-valued ``clingen_allele_id`` +and VEP consequence from the record's authoritative allele. +""" + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import select + +from mavedb.models.variant import Variant +from mavedb.models.variant_annotation_view import VariantAnnotationView +from tests.helpers.constants import TEST_MINIMAL_VARIANT +from tests.helpers.util.annotation import AlleleSpec, seed_mapping_record + + +def _rows_for(session, variant_urn): + # Select explicit columns (not the whole entity): the view is keyless, so an all-null-identity row + # (an unmapped variant) would materialize as a single ``None`` entity. Column rows sidestep that. + stmt = select( + VariantAnnotationView.mapping_record_id, + VariantAnnotationView.hgvs_assay_level, + VariantAnnotationView.hgvs_g, + VariantAnnotationView.hgvs_c, + VariantAnnotationView.hgvs_p, + VariantAnnotationView.clingen_allele_id, + VariantAnnotationView.vep_functional_consequence, + ).where(VariantAnnotationView.variant_urn == variant_urn) + return list(session.execute(stmt).all()) + + +def test_reconstructs_flat_triple_and_authoritative_fields(session, setup_lib_db_with_variant): + """A cdna-assay record with genomic/cdna/protein alleles fills all three HGVS slots; the + authoritative (cdna) allele supplies clingen_allele_id and the VEP consequence.""" + variant = setup_lib_db_with_variant + record = seed_mapping_record( + session, + variant, + assay_level="cdna", + hgvs_assay_level="NM_000001.1:c.1A>T", + alleles=[ + AlleleSpec( + digest="d-cdna", + level="cdna", + is_authoritative=True, + clingen_allele_id="CA123", + hgvs_c="NM_000001.1:c.1A>T", + vep_consequence="missense_variant", + ), + AlleleSpec(digest="d-genomic", level="genomic", hgvs_g="NC_000001.11:g.100A>T"), + AlleleSpec(digest="d-protein", level="protein", hgvs_p="NP_000001.1:p.Met1Leu"), + ], + ) + + rows = _rows_for(session, variant.urn) + assert len(rows) == 1 + row = rows[0] + assert row.mapping_record_id == record.id + assert row.hgvs_assay_level == "NM_000001.1:c.1A>T" + assert row.hgvs_g == "NC_000001.11:g.100A>T" + assert row.hgvs_c == "NM_000001.1:c.1A>T" + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.clingen_allele_id == "CA123" + assert row.vep_functional_consequence == "missense_variant" + + +def test_protein_assay_leaves_nucleotide_slots_null(session, setup_lib_db_with_variant): + """A protein-assay record has only a protein allele: hgvs_p is filled, the nucleotide slots stay + null, and the authoritative protein allele still supplies clingen_allele_id.""" + variant = setup_lib_db_with_variant + seed_mapping_record( + session, + variant, + assay_level="protein", + hgvs_assay_level="NP_000001.1:p.Met1Leu", + alleles=[ + AlleleSpec( + digest="d-prot-only", + level="protein", + is_authoritative=True, + clingen_allele_id="PA9", + hgvs_p="NP_000001.1:p.Met1Leu", + ), + ], + ) + + row = _rows_for(session, variant.urn)[0] + assert row.hgvs_p == "NP_000001.1:p.Met1Leu" + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.clingen_allele_id == "PA9" + + +def test_unmapped_variant_retained_with_null_mapping(session, setup_lib_db_with_score_set): + """The outer join keeps a variant with no live mapping record — it still appears with a null + mapping_record_id and empty mapped fields.""" + variant = Variant( + **TEST_MINIMAL_VARIANT, urn=f"{setup_lib_db_with_score_set.urn}#7", score_set_id=setup_lib_db_with_score_set.id + ) + session.add(variant) + session.commit() + session.refresh(variant) + + row = _rows_for(session, variant.urn)[0] + assert row.mapping_record_id is None + assert row.hgvs_g is None + assert row.hgvs_c is None + assert row.hgvs_p is None + assert row.clingen_allele_id is None + assert row.vep_functional_consequence is None diff --git a/tests/routers/test_statistics.py b/tests/routers/test_statistics.py index 69be6ffba..21ae62676 100644 --- a/tests/routers/test_statistics.py +++ b/tests/routers/test_statistics.py @@ -21,7 +21,12 @@ TEST_PUBMED_IDENTIFIER, VALID_GENE, ) -from tests.helpers.util.score_set import publish_score_set, create_acc_score_set, create_seq_score_set +from tests.helpers.util.score_set import ( + publish_score_set, + create_acc_score_set, + create_seq_score_set, + seed_annotation_substrate, +) from tests.helpers.util.experiment import create_experiment from tests.helpers.util.variant import mock_worker_variant_insertion, create_mapped_variants_for_score_set @@ -65,7 +70,12 @@ def setup_seq_scoreset(setup_router_db, session, data_provider, client, data_fil unpublished_score_set = mock_worker_variant_insertion( client, session, data_provider, unpublished_score_set, data_files / "scores.csv" ) + # Fully-mapped state: target post_mapped_metadata + mapping_state (read by the target-gene + # statistics). MappedVariant rows are seeded here too but are no longer read by the MV. create_mapped_variants_for_score_set(session, unpublished_score_set["urn"], TEST_MINIMAL_MAPPED_VARIANT) + # The published_variants MV reads the MappingRecord/Allele substrate now (not MappedVariant), so + # give each variant a live mapping record for the mapped-variant statistics to count. + seed_annotation_substrate(session, unpublished_score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: publish_score_set(client, unpublished_score_set["urn"]) From 9a3ded6fb4a6a79eb800974124f24149c3a7196d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 21 Jul 2026 23:41:03 -0700 Subject: [PATCH 87/93] perf(gnomad): batch allele lookup in gnomAD linker to end per-row scans link_gnomad_variants_to_alleles queried alleles once per gnomAD variant row inside the loop. The predicate wraps clingen_allele_id in regexp_replace to bridge zero-padded (MaveDB) and stripped (gnomAD dump, #722) CAIDs, which is non-sargable and forces a sequential scan of the alleles table on every execution. Over a linking run this becomes N full scans, saturating database IO (observed as sustained IO:DataFileRead above max vCPUs on the enlarged post-backfill alleles table). Resolve all rows in a single IN scan before the loop and group the result in Python by normalize_caid, so the loop reads from an in-memory map. N scans collapse to one per run, with no schema change. Grouping reuses normalize_caid on both the input and result sides, so the SQL and Python normalization forms are now exercised together. - add test_links_a_batch_of_rows_in_one_pass covering multi-row batching: distinct CAIDs, zero-padding match (#722), and shared-CAID fan-out preserved across a single call --- src/mavedb/lib/gnomad.py | 23 +++++++++++------ tests/lib/test_gnomad.py | 53 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 5fd4d888c..9985e9934 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -1,6 +1,7 @@ import logging import os import re +from collections import defaultdict from enum import Enum from typing import Any, Sequence, Union @@ -188,20 +189,26 @@ def link_gnomad_variants_to_alleles( save_to_logging_context({"num_gnomad_variant_rows": len(gnomad_variant_data)}) logger.debug(msg="Linking gnomAD variants to alleles", extra=logging_context()) + # Resolve all rows' alleles in one query, then group in Python. The regexp_replace predicate + # (normalizing the zero-padded stored CAID to the dump's stripped form, #722) is non-sargable, so + # a per-row query would seq-scan `alleles` once per variant; one IN scan replaces those N scans. + target_caids = {normalize_caid(row.caid) for row in gnomad_variant_data} + alleles_by_caid: dict[str, list[Allele]] = defaultdict(list) + if target_caids: + for allele in db.scalars( + select(Allele).where( + func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2").in_(target_caids) + ) + ).all(): + alleles_by_caid[normalize_caid(allele.clingen_allele_id)].append(allele) + verdicts: dict[int, GnomadLinkVerdict] = {} for index, row in enumerate(gnomad_variant_data, start=1): logger.info( msg=f"Processing gnomAD variant row {index}/{len(gnomad_variant_data)}: {row.caid}", extra=logging_context() ) - # Match on the unpadded CAID: the dump's caid is already stripped, while the stored CAID may - # be zero-padded, so normalize both sides (issue #722). regexp_replace mirrors normalize_caid. - alleles_with_caid = db.scalars( - select(Allele).where( - func.regexp_replace(Allele.clingen_allele_id, _CAID_LEADING_ZERO_RE, r"\1\2") - == normalize_caid(row.caid) - ) - ).all() + alleles_with_caid = alleles_by_caid.get(normalize_caid(row.caid), []) if not alleles_with_caid: continue diff --git a/tests/lib/test_gnomad.py b/tests/lib/test_gnomad.py index f9c8b17aa..b3dc62c28 100644 --- a/tests/lib/test_gnomad.py +++ b/tests/lib/test_gnomad.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from sqlalchemy import select @@ -22,6 +22,7 @@ from tests.helpers.constants import ( TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, + TEST_MAVEDB_ATHENA_ROW, ) ### Tests for gnomad_identifier function ### @@ -148,6 +149,18 @@ def _make_allele(session, caid, *, vrs_digest, level="genomic"): return allele +def _make_gnomad_row(**overrides): + """Build an Athena-style gnomAD row Mock from the canonical test row, applying overrides. + + Mirrors the ``mocked_gnomad_variant_row`` fixture but takes overrides so a test can construct a + batch of distinct rows (different CAIDs, loci) in one call. + """ + row = Mock() + for key, value in {**TEST_MAVEDB_ATHENA_ROW, **overrides}.items(): + setattr(row, key, value) + return row + + def _live_links_for(session, allele_id): return session.scalars( select(GnomadAlleleLink).where( @@ -324,6 +337,44 @@ def test_links_allele_when_dump_strips_leading_zero_from_caid(session, mocked_gn assert len(_live_links_for(session, allele.id)) == 1 +def test_links_a_batch_of_rows_in_one_pass(session): + """Multiple rows in one call are resolved by a single batched allele lookup, then grouped by + normalized CAID. Exercises everything the batching must preserve at once: distinct CAIDs each + reach their own allele, a zero-padded stored CAID still matches a dump-stripped row (#722), and a + CAID shared across alleles fans its variant out to each — all without a per-row query.""" + # Row A: allele stored zero-padded, dump delivers the stripped form. + allele_a = _make_allele(session, "CA025094", vrs_digest="vrs-a") + row_a = _make_gnomad_row(caid="CA25094") + row_a.__setattr__("locus.position", "87961093") # identifier 10-87961093-A-G + + # Row B: a different CAID at a different locus, shared by two alleles (cross-score-set dedup). + allele_b1 = _make_allele(session, "CA341478553", vrs_digest="vrs-b1") + allele_b2 = _make_allele(session, "CA341478553", vrs_digest="vrs-b2", level="cdna") + row_b = _make_gnomad_row(caid="CA341478553") + row_b.__setattr__("locus.position", "87961094") # identifier 10-87961094-A-G + + with patch("mavedb.lib.gnomad.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + result = link_gnomad_variants_to_alleles(session, [row_a, row_b]) + assert result == { + allele_a.id: GnomadLinkVerdict.CREATED, + allele_b1.id: GnomadLinkVerdict.CREATED, + allele_b2.id: GnomadLinkVerdict.CREATED, + } + session.commit() + + # Each allele ends with exactly one live link. + for allele in (allele_a, allele_b1, allele_b2): + assert len(_live_links_for(session, allele.id)) == 1 + + # Two distinct gnomAD variants (one per row's identifier); B's two alleles share row B's variant. + assert len(session.scalars(select(GnomADVariant)).all()) == 2 + b1_variant = _live_links_for(session, allele_b1.id)[0].gnomad_variant_id + b2_variant = _live_links_for(session, allele_b2.id)[0].gnomad_variant_id + a_variant = _live_links_for(session, allele_a.id)[0].gnomad_variant_id + assert b1_variant == b2_variant + assert a_variant != b1_variant + + def test_returns_empty_map_when_no_alleles_match(session, mocked_gnomad_variant_row): result = link_gnomad_variants_to_alleles(session, [mocked_gnomad_variant_row]) assert result == {} From 0f6ba0b8234e6f5907631229f4bcf746d5237e74 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 15:56:34 -0700 Subject: [PATCH 88/93] feat(scripts): gate unenriched selection on a usable transcript The --select unenriched path enqueues annotate_score_set, whose reverse-translation step needs a transcript. It previously selected any mapped-but-not-enriched score set, so ones with no usable transcript were re-selected on every run and burned worker cycles skipping every variant as transcript_unresolved. - split _needs_enrichment into _mapped_without_enrichment and a new _has_usable_transcript predicate (cdna TargetGeneMapping reference_accession, or a live NP_/XP_ RefSeq protein MappingRecord), mirroring the two transcript sources in reverse_translation.py - require a usable transcript for unenriched eligibility - report the count of mapped score sets skipped for lack of a transcript so they are visibly dropped rather than silently lost --- src/mavedb/scripts/run_score_set_pipelines.py | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index 169f17031..6a7571562 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -9,8 +9,10 @@ ones, whose records carry the old ``mapped_date`` and so need a real re-map under the upgraded pipeline. * ``--select unenriched`` → runs ``annotate_score_set`` over mapped score sets that have not yet had - the reverse-translation fan-out (no derived, non-authoritative live allele link). This is the - post-backfill enrichment half. + the reverse-translation fan-out (no derived, non-authoritative live allele link) **and** have a + transcript reverse translation can use. The transcript gate excludes score sets whose every + variant would skip as ``transcript_unresolved``, which would otherwise be re-selected forever and + waste worker cycles. This is the post-backfill enrichment half. **Throughput — gene-grouped ordering.** Score sets whose variants resolve to the same CAIDs/PAIDs cache the expensive ClinGen CAR resolution (24h TTL) and reuse already-deduplicated alleles. So by @@ -41,16 +43,18 @@ import asyncclick as click from arq import create_pool -from sqlalchemy import and_, not_, select +from sqlalchemy import and_, func, not_, or_, select from sqlalchemy.orm import Session, selectinload from mavedb.db.session import SessionLocal from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.enums.processing_state import ProcessingState +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.score_set import ScoreSet from mavedb.models.target_gene import TargetGene +from mavedb.models.target_gene_mapping import TargetGeneMapping from mavedb.models.user import User from mavedb.models.variant import Variant from mavedb.worker.settings import RedisWorkerSettings @@ -104,7 +108,7 @@ def _needs_mapping(remap_before: datetime.date): return variant_needing_mapping.exists() -def _needs_enrichment(): +def _mapped_without_enrichment(): """Predicate: the score set is mapped (a live MappingRecord) but has no reverse-translation-derived (non-authoritative) live allele link yet.""" has_live_record = ( @@ -126,6 +130,63 @@ def _needs_enrichment(): return and_(has_live_record.exists(), not_(has_derived_link.exists())) +def _has_usable_transcript(): + """Predicate: the score set has a transcript that reverse translation can use. + + Mirrors the two transcript sources in + ``worker/jobs/variant_processing/reverse_translation.py``: a cdna ``TargetGeneMapping``'s + ``reference_accession`` (the mapper-supplied coding transcript), and a RefSeq protein + (``NP_``/``XP_``) ``hgvs_assay_level`` on a live ``MappingRecord`` (resolved ``NP_``→``NM_`` via + UTA). A score set with neither would have every variant skip as ``transcript_unresolved``, so + the unenriched selection excludes it rather than re-queue it forever.""" + has_cdna_transcript = ( + select(TargetGeneMapping.id) + .join(TargetGene, TargetGene.id == TargetGeneMapping.target_gene_id) + .where( + TargetGene.score_set_id == ScoreSet.id, + TargetGeneMapping.alignment_level == SequenceLevel.cdna, + TargetGeneMapping.reference_accession.isnot(None), + ) + ) + # `_` is a LIKE wildcard, so escape it to match the literal underscore in the RefSeq prefix. + has_refseq_protein_record = ( + select(MappingRecord.id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where( + Variant.score_set_id == ScoreSet.id, + MappingRecord.valid_to.is_(None), + or_( + MappingRecord.hgvs_assay_level.like(r"NP\_%", escape="\\"), + MappingRecord.hgvs_assay_level.like(r"XP\_%", escape="\\"), + ), + ) + ) + return or_(has_cdna_transcript.exists(), has_refseq_protein_record.exists()) + + +def _needs_enrichment(): + """Predicate: the score set is mapped-but-not-enriched AND has a usable transcript, so + ``annotate_score_set``'s reverse-translation step has something to translate against.""" + return and_(_mapped_without_enrichment(), _has_usable_transcript()) + + +def count_unenrichable_without_transcript(db: Session, *, only_published: bool = True) -> int: + """Count mapped-but-not-enriched score sets that ``--select unenriched`` skips for lack of a + usable transcript, so the operator sees they were intentionally dropped (not lost).""" + query = ( + select(func.count()) + .select_from(ScoreSet) + .where( + ScoreSet.processing_state == ProcessingState.success, + _mapped_without_enrichment(), + not_(_has_usable_transcript()), + ) + ) + if only_published: + query = query.where(ScoreSet.private.is_(False), ScoreSet.published_date.isnot(None)) + return db.scalar(query) or 0 + + def select_score_sets( db: Session, *, @@ -227,6 +288,11 @@ async def main( ) click.echo(f"{len(score_sets)} score set(s) selected ({phase}) for pipeline '{pipeline}'.") + if phase == "unenriched" and not force: + skipped = count_unenrichable_without_transcript(db, only_published=not include_unpublished) + if skipped: + click.echo(f" ({skipped} mapped score set(s) skipped: no usable transcript for reverse translation.)") + if dry_run: for score_set in score_sets: click.echo(f" {_grouping_key(score_set):<12} {score_set.urn}") From 6f0a8e263da69ee5df34c404abc20ba7686e8573 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 23:13:24 -0700 Subject: [PATCH 89/93] feat(variants): stream variant-details NDJSON, add gnomAD variant links - add preMapped/postMapped VRS pair to VariantDetail and switch GET /variants/{urn} to exclude_none=False so absent fields serialize as null instead of being dropped - add streaming NDJSON GET /score-sets/{urn}/variant-details, replacing the retired GET /score-sets/{urn}/mapped-variants - replace MappedVariant-keyed gnomAD lookups with a substrate-based get_gnomad_variants_with_variant_urns, pairing each gnomAD variant with the score-set variant URNs it links to (GnomADVariantWithVariantLinks) - add as_of time-travel support to the CSV variants export and the gnomad-variants endpoint, echoed via X-As-Of response headers - extract mapped_hgvs_by_level/get_protein_hgvs_by_record helpers out of score_set_variants for reuse by the CSV export --- src/mavedb/lib/gnomad.py | 73 ++++- src/mavedb/lib/score_set_variants.py | 132 +++++---- src/mavedb/lib/variant_detail.py | 7 + src/mavedb/routers/score_sets.py | 234 ++++++++++----- src/mavedb/routers/variants.py | 5 +- src/mavedb/view_models/gnomad_variant.py | 25 +- src/mavedb/view_models/variant_detail.py | 28 +- tests/helpers/constants.py | 2 +- tests/helpers/util/annotation.py | 5 +- tests/routers/test_score_set.py | 358 +++++++++++++++++++---- tests/routers/test_variant.py | 16 +- 11 files changed, 680 insertions(+), 205 deletions(-) diff --git a/src/mavedb/lib/gnomad.py b/src/mavedb/lib/gnomad.py index 9985e9934..830cc62d9 100644 --- a/src/mavedb/lib/gnomad.py +++ b/src/mavedb/lib/gnomad.py @@ -2,8 +2,10 @@ import os import re from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime from enum import Enum -from typing import Any, Sequence, Union +from typing import Any, Optional, Sequence, Union from sqlalchemy import Connection, Row, func, select, text from sqlalchemy.orm import Session @@ -13,6 +15,9 @@ from mavedb.models.allele import Allele from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapping_record import MappingRecord +from mavedb.models.mapping_record_allele import MappingRecordAllele +from mavedb.models.variant import Variant logger = logging.getLogger(__name__) @@ -39,6 +44,72 @@ class GnomadLinkVerdict(str, Enum): UNCHANGED = "unchanged" # a live link already pointed at the resolved variant; left untouched +@dataclass(frozen=True) +class GnomadVariantLink: + """One (score-set variant, annotated allele) pair a gnomAD frequency record reaches. + + Mirrors :class:`mavedb.lib.clinical_controls.ControlVariantLink`. ``allele_digest`` is the VRS digest + of the allele gnomAD's frequency was linked to. + """ + + variant_urn: str + allele_digest: str + + +def get_gnomad_variants_with_variant_urns( + db: Session, + score_set_id: int, + *, + as_of: Optional[datetime] = None, + db_version: Optional[str] = None, +) -> list[tuple[GnomADVariant, list[GnomadVariantLink]]]: + """The live gnomAD frequency records for a score set, each paired with the score-set variants that link + to it. + + Walks ``GnomadAlleleLink → Allele → MappingRecordAllele → MappingRecord → Variant`` with every + ``ValidTime`` hop constrained to the same instant (``as_of``, defaulting to currently-live). Optionally + narrowed to one gnomAD release (``db_version``). Records come back in first-seen order; each + record's links preserve query order. + """ + stmt = ( + select( + GnomADVariant, + Variant.urn.label("variant_urn"), + Allele.vrs_digest.label("allele_digest"), + ) + .join(GnomadAlleleLink, GnomadAlleleLink.gnomad_variant_id == GnomADVariant.id) + .join(Allele, Allele.id == GnomadAlleleLink.allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(MappingRecord, MappingRecord.id == MappingRecordAllele.mapping_record_id) + .join(Variant, Variant.id == MappingRecord.variant_id) + .where(GnomadAlleleLink.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(MappingRecord.live_at(as_of)) + .where(Variant.score_set_id == score_set_id) + .distinct() + ) + if db_version is not None: + stmt = stmt.where(GnomADVariant.db_version == db_version) + + variants: dict[int, GnomADVariant] = {} + links_by_variant: dict[int, list[GnomadVariantLink]] = {} + + for gnomad_variant, variant_urn, allele_digest in db.execute(stmt).tuples(): + # TODO#372: nullable URN + if variant_urn is None: + continue + + if gnomad_variant.id not in variants: + variants[gnomad_variant.id] = gnomad_variant + links_by_variant[gnomad_variant.id] = [] + + links_by_variant[gnomad_variant.id].append( + GnomadVariantLink(variant_urn=variant_urn, allele_digest=allele_digest) + ) + + return [(gnomad_variant, links_by_variant[gnomad_variant.id]) for gnomad_variant in variants.values()] + + def gnomad_identifier(contig: str, position: Union[str, int], alleles: list[str]) -> str: """ Generate a gnomAD variant identifier based on contig, position, and alleles. diff --git a/src/mavedb/lib/score_set_variants.py b/src/mavedb/lib/score_set_variants.py index f0c75390f..c1f45c545 100644 --- a/src/mavedb/lib/score_set_variants.py +++ b/src/mavedb/lib/score_set_variants.py @@ -118,43 +118,104 @@ def _hgvs_field_for_str(hgvs: Optional[str]) -> Optional[HgvsField]: return HgvsField(hgvs=hgvs, position=block.position, ref=block.ref, alt=block.alt) -def _mapped_triple( +def mapped_hgvs_by_level( *, assay_level: Optional[SequenceLevel], assay_level_hgvs: Optional[str], sibling_level: Optional[str], sibling_hgvs: Optional[str], protein_hgvs: Optional[str], -) -> MappedTriple: - """Assemble the canonical :class:`MappedTriple` for one variant from its per-row query fields. +) -> dict[str, Optional[str]]: + """The canonical mapped HGVS **string** at each populated level slot, keyed by ``SequenceLevel`` value. - - Measured slot (``mapped[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). + The canonical projection of a measured allele onto various levels relative to its reference frame: + - Measured slot (``[assay_level]``) ← ``assay_level_hgvs`` (the record's mapped assay HGVS). - Other nucleotide slot ← the authoritative link's ``projection_group`` sibling (``sibling_level`` / ``sibling_hgvs``), populated only on a nucleotide assay and only where the pairing is recorded. - ``protein`` slot ← the separate protein-apex subquery. A **protein** assay fills only ``protein`` (the measured slot is protein itself); its nucleotide - fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately left ``None``. An unmapped variant - (``assay_level is None``) yields an empty triple. + fan-out is ambiguous, so ``cdna``/``genomic`` are deliberately absent. An unmapped variant + (``assay_level is None``) yields an empty mapping. """ - slots: dict[str, Optional[HgvsField]] = {} - # Protein assay: the measured slot *is* protein. No canonical c/g — do not fabricate one. + slots: dict[str, Optional[str]] = {} if assay_level == SequenceLevel.protein: - slots[SequenceLevel.protein.value] = _hgvs_field_for_str(assay_level_hgvs) + slots[SequenceLevel.protein.value] = assay_level_hgvs elif assay_level in (SequenceLevel.cdna, SequenceLevel.genomic): - slots[assay_level.value] = _hgvs_field_for_str(assay_level_hgvs) + slots[assay_level.value] = assay_level_hgvs + slots[SequenceLevel.protein.value] = protein_hgvs # The other nucleotide level, from the projection_group sibling (null where unpopulated). if sibling_level is not None: - slots[sibling_level] = _hgvs_field_for_str(sibling_hgvs) - slots[SequenceLevel.protein.value] = _hgvs_field_for_str(protein_hgvs) + slots[sibling_level] = sibling_hgvs + + return slots + +def _mapped_triple( + *, + assay_level: Optional[SequenceLevel], + assay_level_hgvs: Optional[str], + sibling_level: Optional[str], + sibling_hgvs: Optional[str], + protein_hgvs: Optional[str], +) -> MappedTriple: + """Assemble the canonical :class:`MappedTriple` for one variant, wrapping each slot from + :func:`mapped_hgvs_by_level` as an :class:`HgvsField`.""" + slots = mapped_hgvs_by_level( + assay_level=assay_level, + assay_level_hgvs=assay_level_hgvs, + sibling_level=sibling_level, + sibling_hgvs=sibling_hgvs, + protein_hgvs=protein_hgvs, + ) return MappedTriple( - genomic=slots.get(SequenceLevel.genomic.value), - cdna=slots.get(SequenceLevel.cdna.value), - protein=slots.get(SequenceLevel.protein.value), + genomic=_hgvs_field_for_str(slots.get(SequenceLevel.genomic.value)), + cdna=_hgvs_field_for_str(slots.get(SequenceLevel.cdna.value)), + protein=_hgvs_field_for_str(slots.get(SequenceLevel.protein.value)), ) +def get_protein_hgvs_by_record(db: Session, score_set_id: int, *, as_of: Optional[datetime] = None) -> dict[int, str]: + """The canonical protein-level mapped HGVS for each of a score set's live mapping records, keyed by + ``mapping_record.id``. Records with no protein projection (UTR/intronic) are simply absent from the + map. Shared by the lean whole-set view and the CSV export so both resolve the protein slot the same way. + + This runs as its **own** query, never a join into a per-variant projection. Joined in, it forces an + O(N^2) plan: the ``DISTINCT ON`` cardinality is opaque to the planner, so it lands on the inner side + of a nested loop and is rescanned in full once per variant — historically ~97% of the lean endpoint's + total time. Pulling it out and stitching by ``mapping_record_id`` in Python keeps both queries O(N). + """ + prot_link = aliased(MappingRecordAllele) + prot_allele = aliased(Allele) + prot_record = aliased(MappingRecord) + prot_variant = aliased(Variant) + protein_statement = ( + select( + prot_link.mapping_record_id.label("mapping_record_id"), + prot_allele.hgvs_p.label("protein_hgvs"), + ) + # DISTINCT ON -> at most one protein level row per record. + .distinct(prot_link.mapping_record_id) + # link -> allele: level and hgvs_p live on the *allele*. + .join(prot_allele, prot_allele.id == prot_link.allele_id) + # link -> record -> variant: the bridge to scoreset_id, to scope the query. + .join(prot_record, prot_record.id == prot_link.mapping_record_id) + .join(prot_variant, prot_variant.id == prot_record.variant_id) + # Scope to this score set so the cost scales with the response. + .where(prot_variant.score_set_id == score_set_id) + # Live links only. A live link implies a live parent record (ValidTime invariant), so this also + # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. + .where(prot_link.live_at(as_of)) + # Only the protein-level allele. + .where(prot_allele.level == SequenceLevel.protein.value) + # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives + # per record — inert today (only one protein allele exists). + .order_by(prot_link.mapping_record_id, prot_allele.id) + ) + # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). + return {row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement)} + + def get_lean_score_set_variants( db: Session, score_set: ScoreSet, *, as_of: Optional[datetime] = None ) -> list[LeanVariantRecord]: @@ -236,44 +297,9 @@ def get_lean_score_set_variants( .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) ) - # The protein projection runs as its own query, NOT a join into the base statement. Joined in, it forces - # an O(N^2) plan: the DISTINCT ON cardinality is opaque to the planner (est. ~18 vs ~11.5k actual), so it - # buried the projection on the inner side of a nested loop and rescanned it in full once per variant — - # ~97% of the endpoint's total time. MATERIALIZED in the query above fixes the *construction* but not the - # per-variant *rescan*. Pulling out and stitching these queries by mapping_record_id in Python enables both - # queries to run in O(N) time, making this function (and the endpoint it serves) considerably faster. - prot_link = aliased(MappingRecordAllele) - prot_allele = aliased(Allele) - prot_record = aliased(MappingRecord) - prot_variant = aliased(Variant) - protein_statement = ( - select( - prot_link.mapping_record_id.label("mapping_record_id"), - prot_allele.hgvs_p.label("protein_hgvs"), - ) - # DISTINCT ON -> at most one protein level row per record. - .distinct(prot_link.mapping_record_id) - # link -> allele: level and hgvs_p live on the *allele*. - .join(prot_allele, prot_allele.id == prot_link.allele_id) - # link -> record -> variant: the bridge to scoreset_id, to scope the query. - .join(prot_record, prot_record.id == prot_link.mapping_record_id) - .join(prot_variant, prot_variant.id == prot_record.variant_id) - # Scope to this score set so the cost scales with the response. - .where(prot_variant.score_set_id == score_set.id) - # Live links only. A live link implies a live parent record (ValidTime invariant), so this also - # covers record-liveness — no prot_record.live_at needed; the MR join is just the scoping bridge. - .where(prot_link.live_at(as_of)) - # Only the protein-level allele. - .where(prot_allele.level == SequenceLevel.protein.value) - # DISTINCT ON requires ORDER BY to lead with its column; allele.id then picks which row survives - # per record — inert today (only one protein allele exists). - .order_by(prot_link.mapping_record_id, prot_allele.id) - ) - # Keyed by record id (a globally-unique PK -> the right protein row, no cross-set risk). Some records - # have no protein projection (UTR/intronic) -> absent from the map -> null protein field below. - protein_hgvs_by_record: dict[int, str] = { - row.mapping_record_id: row.protein_hgvs for row in db.execute(protein_statement) - } + # The protein projection runs as its own query, NOT a join into the base statement (see + # get_protein_hgvs_by_record for why), stitched back by mapping_record_id below. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) return [ LeanVariantRecord( diff --git a/src/mavedb/lib/variant_detail.py b/src/mavedb/lib/variant_detail.py index 567fcf226..cf9a95cc0 100644 --- a/src/mavedb/lib/variant_detail.py +++ b/src/mavedb/lib/variant_detail.py @@ -70,6 +70,11 @@ class VariantDetail: assay_level_digest: Optional[str] clingen_allele_id: Optional[str] + # The raw GA4GH VRS pair, surfaced flat so a bulk/VRS consumer reads them directly rather than + # digging the measured allele out of the Cat-VRS below. + pre_mapped: Optional[dict[str, Any]] + post_mapped: Optional[dict[str, Any]] + # Spec-pure GA4GH Cat-VRS (no MaveDB fields inside), plus the MaveDB layer riding alongside: the # per-allele identity sidecar, keyed by VRS digest — one entry per linked allele (the record-scoped, # all-levels set, sharing keys with `annotations`), carrying level + HGVS + ClinGen id + relation. @@ -258,6 +263,8 @@ def get_variant_detail( reference_hgvs=reference_hgvs, assay_level_digest=assay_level_digest, clingen_allele_id=clingen_allele_id, + pre_mapped=record.pre_mapped if record is not None else None, + post_mapped=authoritative.allele.post_mapped if authoritative is not None else None, molecular_representation=molecular_representation, mode=mode, alleles=alleles, diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 7baeb0893..6bbed79f8 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -38,6 +38,7 @@ from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets +from mavedb.lib.gnomad import get_gnomad_variants_with_variant_urns from mavedb.lib.identifiers import ( create_external_gene_identifier_offset, find_or_create_doi_identifier, @@ -71,13 +72,12 @@ generate_experiment_urn, generate_score_set_urn, ) +from mavedb.lib.variant_detail import get_variant_detail from mavedb.lib.workflow.kickoff import enqueue_pipeline_for_score_set from mavedb.models.contributor import Contributor from mavedb.models.enums.processing_state import ProcessingState from mavedb.models.experiment import Experiment -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.license import License -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession @@ -92,7 +92,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) -from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set +from mavedb.view_models import clinical_control, gnomad_variant, score_set from mavedb.view_models.contributor import ContributorCreate from mavedb.view_models.doi_identifier import DoiIdentifierCreate from mavedb.view_models.lean_variant import LeanVariant @@ -100,6 +100,7 @@ from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata from mavedb.view_models.search import ScoreSetsSearch, ScoreSetsSearchFilterOptionsResponse, ScoreSetsSearchResponse from mavedb.view_models.target_gene import TargetGeneCreate +from mavedb.view_models.variant_detail import VariantDetail TAG_NAME = "Score Sets" logger = logging.getLogger(__name__) @@ -869,6 +870,108 @@ async def get_score_set_lean_variants( return get_lean_score_set_variants(db, score_set, as_of=as_of) +def _stream_score_set_variant_details( + db: Session, + variants: Sequence[Variant], + superseding_score_set: Optional[ScoreSet], + visible_calibration_ids: set[int], + *, + as_of: Optional[datetime] = None, +): + """Serialize the whole-set variant-detail export as NDJSON — one :class:`VariantDetail` per line. + + Assembles each variant's detail envelope (the flat assay fields + ``preMapped``/``postMapped`` VRS + pair + the spec-pure GA4GH CategoricalVariant + the digest-keyed VEP/gnomAD/ClinVar annotation map) + one at a time, so a large score set streams rather than building every envelope up front. + ``superseding_score_set`` / ``visible_calibration_ids`` are resolved once for the whole set and + threaded into every per-variant build. + """ + for variant in variants: + detail = get_variant_detail( + db, + variant, + superseding_score_set=superseding_score_set, + visible_calibration_ids=visible_calibration_ids, + as_of=as_of, + ) + # get_variant_detail returns the lib transit dataclass; coerce it through the view model. + # exclude_none=False keeps a stable key set per line for programmatic consumers. + yield VariantDetail.model_validate(detail).model_dump_json(by_alias=True, exclude_none=False) + "\n" + + +@router.get( + "/score-sets/{urn}/variant-details", + status_code=200, + responses={ + 200: { + "content": {"application/x-ndjson": {}}, + "description": ( + "Newline-delimited JSON: one VariantDetail per mapped variant — the same envelope the " + "single-variant GET /variants/{urn} route serves, carrying the flat preMapped/postMapped " + "VRS pair, the spec-pure GA4GH CategoricalVariant, and the digest-keyed VEP/gnomAD/ClinVar " + "annotation map." + ), + }, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Download a score set's variant details (VRS + Cat-VRS + annotations)", +) +async def get_score_set_variant_details( + *, + urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the molecular layer (VRS, Cat-VRS membership + VEP/gnomAD/ClinVar annotations) " + "as it stood at this instant, over the score set's fixed scores. ISO 8601, ideally " + "timezone-aware. Content valid-time only — it never re-selects a score-set version. Defaults " + "to current." + ), + ), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """Download the score set's variant details — the whole-set streaming pair of the single-variant + ``GET /variants/{urn}`` detail endpoint, and the substrate-faithful replacement for the retired + ``/mapped-variants`` export. + + One record per *mapped* variant (unmapped variants carry no VRS and are omitted): the same + VariantDetail envelope the single-variant route serves — the flat ``preMapped``/``postMapped`` VRS + pair for VRS consumers, plus the spec-pure GA4GH CategoricalVariant and the digest-keyed + VEP/gnomAD/ClinVar annotation map for the full molecular picture. + + Streamed as NDJSON (like the annotated-variant exports) so a large score set downloads without + materializing every envelope server-side and a client can process it line by line. ``as_of`` + time-travels the molecular layer only (scores/classifications are immutable); the resolved value is + echoed in ``X-As-Of`` and the variant count in ``X-Total-Count``. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "variant-details", "as_of": as_of}) + + score_set = await fetch_score_set_by_urn(db, urn, user_data, None, False) + # fetch_score_set_by_urn already blanks a non-readable superseding version and filters + # score_calibrations to the readable ones, so both are visibility-resolved here. + visible_calibration_ids = {sc.id for sc in score_set.score_calibrations if sc.id is not None} + variants = get_annotatable_variants(db, score_set, as_of=as_of) + return StreamingResponse( + _stream_score_set_variant_details( + db, + variants, + score_set.superseding_score_set, + visible_calibration_ids, + as_of=as_of, + ), + media_type="application/x-ndjson", + headers={ + "X-As-Of": as_of.isoformat() if as_of is not None else "current", + "X-Total-Count": str(len(variants)), + "X-Processing-Started": datetime.now().isoformat(), + "X-Stream-Type": "variant-detail", + "Access-Control-Expose-Headers": "X-As-Of, X-Total-Count, X-Processing-Started, X-Stream-Type", + }, + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, @@ -899,6 +1002,14 @@ def get_score_set_variants_csv( drop_na_columns: Optional[bool] = None, include_custom_columns: Optional[bool] = None, include_post_mapped_hgvs: Optional[bool] = None, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this " + "instant, over the variant's immutable submitted HGVS/scores/counts. ISO 8601, ideally " + "timezone-aware. No effect on the scores/counts namespaces. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -940,6 +1051,7 @@ def get_score_set_variants_csv( "start": start, "limit": limit, "drop_na_columns": drop_na_columns, + "as_of": as_of, } ) @@ -982,8 +1094,13 @@ def get_score_set_variants_csv( drop_na_columns, include_custom_columns, include_post_mapped_hgvs, + as_of, + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"X-As-Of": as_of.isoformat() if as_of is not None else "current"}, ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") @router.get( @@ -1102,51 +1219,6 @@ async def get_score_set_counts_csv( return StreamingResponse(iter([csv_str]), media_type="text/csv") -@router.get( - "/score-sets/{urn}/mapped-variants", - status_code=200, - response_model=list[mapped_variant.MappedVariant], - responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - summary="Get mapped variants from score set by URN", -) -def get_score_set_mapped_variants( - *, - urn: str, - db: Session = Depends(deps.get_db), - user_data: Optional[UserData] = Depends(get_current_user), -) -> list[MappedVariant]: - """ - Return mapped variants from a score set, identified by URN. - """ - save_to_logging_context({"requested_resource": urn, "resource_property": "mapped-variants"}) - - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() - if not score_set: - logger.info( - msg="Could not fetch the requested mapped variants; No such score set exist.", extra=logging_context() - ) - raise HTTPException(status_code=404, detail=f"score set with URN {urn} not found") - - assert_permission(user_data, score_set, Action.READ) - - mapped_variants = ( - db.query(MappedVariant) - .filter(ScoreSet.urn == urn) - .filter(ScoreSet.id == Variant.score_set_id) - .filter(Variant.id == MappedVariant.variant_id) - .all() - ) - - if not mapped_variants: - logger.info(msg="No mapped variants are associated with the requested score set.", extra=logging_context()) - raise HTTPException( - status_code=404, - detail=f"No mapped variant associated with score set URN {urn} was found", - ) - - return mapped_variants - - def _stream_generated_annotations(db, variants, annotation_function, as_of=None): """ Generator function to stream annotations as pure NDJSON data. @@ -2587,7 +2659,7 @@ async def get_clinical_controls_options_for_score_set( @router.get( "/score-sets/{urn}/gnomad-variants", status_code=200, - response_model=list[gnomad_variant.GnomADVariantWithMappedVariants], + response_model=list[gnomad_variant.GnomADVariantWithVariantLinks], response_model_exclude_none=True, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, summary="Get gnomad variants for a score set", @@ -2595,16 +2667,26 @@ async def get_clinical_controls_options_for_score_set( async def get_gnomad_variants_for_score_set( *, urn: str, + response: Response, + as_of: Optional[datetime] = Query( + default=None, + description=( + "Reconstruct the allele → gnomAD link state as it stood at this instant. " + "ISO 8601, ideally timezone-aware. Defaults to current." + ), + ), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), version: Optional[str] = None, -) -> Sequence[GnomADVariant]: +) -> list[gnomad_variant.GnomADVariantWithVariantLinks]: """ - Fetch relevant gnomad variants for a given score set. + Fetch relevant gnomad variants for a given score set, each paired with the score-set variants (and + annotated allele digests) it links to over the allele substrate. """ - save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants"}) + save_to_logging_context({"requested_resource": urn, "resource_property": "gnomad_variants", "as_of": as_of}) + response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" - # Rename user facing kwargs for consistency with code base naming conventions. My-py doesn't care for us redefining db. + # Rename user facing kwargs for consistency with code base naming conventions. db_version = version item: Optional[ScoreSet] = db.scalars(select(ScoreSet).where(ScoreSet.urn == urn)).one_or_none() @@ -2617,28 +2699,12 @@ async def get_gnomad_variants_for_score_set( assert_permission(user_data, item, Action.READ) - gnomad_variants_query = ( - select(GnomADVariant) - .join(MappedVariant, GnomADVariant.mapped_variants) - .join(Variant) - .where(Variant.score_set_id == item.id) - ) - if db_version is not None: save_to_logging_context({"db_version": db_version}) - gnomad_variants_query = gnomad_variants_query.where(GnomADVariant.db_version == db_version) - gnomad_variants_for_item: Sequence[GnomADVariant] = db.scalars(gnomad_variants_query).all() - gnomad_variants_with_mapped_variant = [] - for gnomad_variant_in_item in gnomad_variants_for_item: - gnomad_variant_in_item.mapped_variants = [ - mv for mv in gnomad_variant_in_item.mapped_variants if mv.current and mv.variant.score_set_id == item.id - ] - - if gnomad_variant_in_item.mapped_variants: - gnomad_variants_with_mapped_variant.append(gnomad_variant_in_item) + gnomad_variants = get_gnomad_variants_with_variant_urns(db, item.id, as_of=as_of, db_version=db_version) - if not gnomad_variants_with_mapped_variant: + if not gnomad_variants: logger.info( msg="No gnomad variants matching the provided filters are associated with the requested score set.", extra=logging_context(), @@ -2648,6 +2714,26 @@ async def get_gnomad_variants_for_score_set( detail=f"No gnomad variants matching the provided filters associated with score set URN {urn} were found", ) - save_to_logging_context({"resource_count": len(gnomad_variants_for_item)}) + save_to_logging_context({"resource_count": len(gnomad_variants)}) - return gnomad_variants_for_item + return [ + gnomad_variant.GnomADVariantWithVariantLinks.model_validate( + { + "id": gv.id, + "db_name": gv.db_name, + "db_identifier": gv.db_identifier, + "db_version": gv.db_version, + "allele_count": gv.allele_count, + "allele_number": gv.allele_number, + "allele_frequency": gv.allele_frequency, + "faf95_max": gv.faf95_max, + "faf95_max_ancestry": gv.faf95_max_ancestry, + "creation_date": gv.creation_date, + "modification_date": gv.modification_date, + "variant_links": [ + {"variant_urn": link.variant_urn, "allele_digest": link.allele_digest} for link in links + ], + } + ) + for gv, links in gnomad_variants + ] diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index 78144594c..70e4bf788 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -136,7 +136,10 @@ def lookup_variants_by_vrs_identifier( status_code=200, response_model=VariantDetail, responses={**ACCESS_CONTROL_ERROR_RESPONSES}, - response_model_exclude_none=True, + # Emit nulls rather than omitting absent fields: this shape is a stable, self-describing envelope + # (its bulk pair, GET /score-sets/{urn}/variant-details, streams the same object one-per-line for + # dataframe/columnar consumption), so every response carries the same key set. + response_model_exclude_none=False, summary="Fetch assayed variant detail by URN", ) def get_variant( diff --git a/src/mavedb/view_models/gnomad_variant.py b/src/mavedb/view_models/gnomad_variant.py index 97dd675e6..116a4e21b 100644 --- a/src/mavedb/view_models/gnomad_variant.py +++ b/src/mavedb/view_models/gnomad_variant.py @@ -8,7 +8,7 @@ from mavedb.view_models.base.base import BaseModel if TYPE_CHECKING: - from mavedb.view_models.mapped_variant import MappedVariant, MappedVariantCreate, SavedMappedVariant + from mavedb.view_models.mapped_variant import MappedVariantCreate class GnomADVariantBase(BaseModel): @@ -54,10 +54,21 @@ class Config: from_attributes = True -class SavedGnomADVariantWithMappedVariants(SavedGnomADVariant): - """Saved GnomAD variant records with mapped variants.""" +class GnomadVariantLink(BaseModel): + """One score-set variant a gnomAD frequency record reaches, tagged with the annotated allele's digest. - mapped_variants: Sequence["SavedMappedVariant"] + Mirrors :class:`clinical_control.ClinvarVariantLink`: a gnomAD variant fans out to every allele that + resolved to it, and each allele belongs to a score-set variant. + """ + + variant_urn: str + allele_digest: Optional[str] = None + + +class SavedGnomADVariantWithVariantLinks(SavedGnomADVariant): + """Saved gnomAD variant paired with the score-set variants (and annotated allele digests) it links to.""" + + variant_links: Sequence[GnomadVariantLink] class GnomADVariant(SavedGnomADVariant): @@ -66,7 +77,7 @@ class GnomADVariant(SavedGnomADVariant): pass -class GnomADVariantWithMappedVariants(SavedGnomADVariantWithMappedVariants): - """GnomAD variant view model with mapped variants for non-admin clients.""" +class GnomADVariantWithVariantLinks(SavedGnomADVariantWithVariantLinks): + """GnomAD variant + its score-set variant links, for non-admin clients.""" - mapped_variants: Sequence["MappedVariant"] + pass diff --git a/src/mavedb/view_models/variant_detail.py b/src/mavedb/view_models/variant_detail.py index d9bc08344..9603bc7b3 100644 --- a/src/mavedb/view_models/variant_detail.py +++ b/src/mavedb/view_models/variant_detail.py @@ -35,15 +35,22 @@ class VariantDetail(BaseModel): """The assayed variant-detail envelope (``GET /variants/{urn}``). Two tiers: flat, UI-ergonomic assay fields (the ``targetHgvs``/``referenceHgvs`` coordinate pair - is a client-side toggle, no refetch) plus the spec-pure GA4GH ``molecularRepresentation`` - (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer rides alongside, keyed by VRS - digest: the ``alleles`` identity sidecar (per-allele ``level`` / ``hgvs`` / ``clingenAlleleId`` / - ``relation`` — one entry per linked allele, sharing keys with ``annotations``) and the - ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a superseded variant self-describe: - ``supersededByScoreSet`` is the superseding *score set*'s URN, not a variant URN. Supersession is - versioned at the score-set level, and a newer version may add, drop, or renumber variants — so there - is no stable superseding-*variant* pointer to hand back; a consumer resolves the current measurement - by looking this variant up within that score set. Absent fields are omitted. + is a client-side toggle, no refetch; the ``preMapped``/``postMapped`` raw VRS pair lets a + VRS/bulk consumer read the assayed-level and measured VRS directly) plus the spec-pure GA4GH + ``molecularRepresentation`` (``CategoricalVariant``, no MaveDB fields inside). The MaveDB layer + rides alongside, keyed by VRS digest: the ``alleles`` identity sidecar (per-allele ``level`` / + ``hgvs`` / ``clingenAlleleId`` / ``relation`` — one entry per linked allele, sharing keys with + ``annotations``) and the ``annotations`` map. ``isCurrent``/``supersededByScoreSet`` let a + superseded variant self-describe: ``supersededByScoreSet`` is the superseding *score set*'s URN, + not a variant URN. Supersession is versioned at the score-set level, and a newer version may add, + drop, or renumber variants — so there is no stable superseding-*variant* pointer to hand back; a + consumer resolves the current measurement by looking this variant up within that score set. + + Unlike most MaveDB response models, this one serializes with ``exclude_none=False`` on both routes + that emit it (``GET /variants/{urn}`` and the bulk ``GET /score-sets/{urn}/variant-details`` NDJSON + stream): the shape is a stable, self-describing envelope, so every record carries the same key set + and an unmapped variant reads ``preMapped``/``postMapped``/``molecularRepresentation`` as ``null`` + rather than dropping them. The two routes are kept in lockstep — the same object, one shape. """ urn: str @@ -57,6 +64,9 @@ class VariantDetail(BaseModel): assay_level_digest: Optional[str] = None clingen_allele_id: Optional[str] = None + pre_mapped: Optional[dict[str, Any]] = None + post_mapped: Optional[dict[str, Any]] = None + molecular_representation: Optional[dict[str, Any]] = None mode: Optional[str] = None alleles: dict[str, AlleleIdentity] = {} diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index 8766c2f53..7f00ed5f8 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -2147,7 +2147,7 @@ "faf95MaxAncestry": TEST_GNOMAD_FAF95_MAX_ANCESTRY, "creationDate": date.today().isoformat(), "modificationDate": date.today().isoformat(), - "recordType": "GnomADVariantWithMappedVariants", + "recordType": "GnomADVariantWithVariantLinks", "id": 1, # Presuming this is the only gnomAD variant in the database } diff --git a/tests/helpers/util/annotation.py b/tests/helpers/util/annotation.py index f845cdc1c..24b9c11f3 100644 --- a/tests/helpers/util/annotation.py +++ b/tests/helpers/util/annotation.py @@ -60,11 +60,13 @@ def seed_mapping_record( assay_level: str = "cdna", hgvs_assay_level: Optional[str] = None, mapping_api_version: str = "test.0.0", + pre_mapped: Optional[dict] = None, valid_from: Optional[datetime] = None, ) -> MappingRecord: """Give ``variant`` a live mapping record carrying ``alleles`` and their annotations. - ``variant`` may be a ``Variant`` instance or its URN. When ``valid_from`` is set it stamps the + ``variant`` may be a ``Variant`` instance or its URN. ``pre_mapped`` sets the record's assayed-level + VRS (the flat ``preMapped`` field on the variant detail). When ``valid_from`` is set it stamps the record, its allele links, and every annotation link, so ``as_of`` reconstruction tests can place the whole graph at a chosen instant (absent it, ``valid_from`` defaults to ``now()`` and the row is live). Returns the created ``MappingRecord``. @@ -79,6 +81,7 @@ def seed_mapping_record( assay_level=assay_level, hgvs_assay_level=hgvs_assay_level, mapping_api_version=mapping_api_version, + pre_mapped=pre_mapped, ) if valid_from is not None: record.valid_from = valid_from diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 77150ba4c..2bc830db3 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -25,6 +25,7 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel from mavedb.view_models.lean_variant import LeanVariant @@ -61,6 +62,8 @@ TEST_SAVED_GNOMAD_VARIANT, TEST_SAVED_TAXONOMY, TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider @@ -82,14 +85,12 @@ create_seq_score_set_with_mapped_variants, create_seq_score_set_with_variants, link_clinical_controls_to_alleles, - link_clinvar_control_to_mapped_variant, - link_gnomad_variants_to_mapped_variants, publish_score_set, seed_annotation_substrate, + seed_csv_substrate, ) from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import ( - create_mapped_variants_for_score_set, mock_worker_variant_insertion, ) @@ -3399,7 +3400,17 @@ def test_download_variants_data_file( score_set = create_seq_score_set(client, experiment["urn"]) score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3584,7 +3595,17 @@ def test_download_scores_counts_and_post_mapped_variants_file( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) if mapped_variant is not None: - create_mapped_variants_for_score_set(session, score_set["urn"], mapped_variant) + if has_hgvs_g: + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=mapped_variant["hgvs_g"], + hgvs_c=mapped_variant["hgvs_c"], + hgvs_p=mapped_variant["hgvs_p"], + ) + elif has_hgvs_p: + seed_csv_substrate(session, score_set, assay_level="protein", hgvs_p=mapped_variant["hgvs_p"]) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3622,8 +3643,14 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + # Seed mapped alleles with a VEP consequence on the authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3646,13 +3673,15 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants then set ClinGen allele id for first mapped variant - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles, with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3670,11 +3699,31 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie def test_download_gnomad_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) - # Link a gnomAD variant to the first mapped variant (version may not match export filter) - score_set = create_seq_score_set_with_mapped_variants( - client, session, data_provider, experiment["urn"], data_files / "scores.csv" + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # The CSV gnomAD column reflects the served release (v4.1); seed one and link it to the first variant's + # authoritative allele. + gnomad_variant = GnomADVariantDbModel( + db_name="gnomAD", + db_identifier="10-1-A-G", + db_version="v4.1", + allele_count=3, + allele_number=1613510, + allele_frequency=3 / 1613510, + faf95_max=6.8e-07, + faf95_max_ancestry="nfe", + ) + session.add(gnomad_variant) + session.commit() + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + gnomad_variant_ids=[gnomad_variant.id], + annotate="first", ) - link_gnomad_variants_to_mapped_variants(session, score_set) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3686,6 +3735,9 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames + rows = list(reader) + assert rows[0]["gnomad.gnomad_af"] == str(3 / 1613510) + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows[1:]) def test_download_clingen_and_vep_file_in_variant_data_path( @@ -3696,13 +3748,16 @@ def test_download_clingen_and_vep_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a VEP consequence (all variants) and a ClinGen id (first variant only). + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3728,13 +3783,15 @@ def test_download_clingen_and_scores_file_in_variant_data_path( score_set = mock_worker_variant_insertion( client, session, data_provider, score_set, data_files / "scores.csv", data_files / "counts.csv" ) - # Create mapped variants with VEP consequence populated - create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) - db_score_set = session.query(ScoreSetDbModel).filter(ScoreSetDbModel.urn == score_set["urn"]).one() - first_mapped_variant = db_score_set.variants[0].mapped_variants[0] - first_mapped_variant.clingen_allele_id = VALID_CLINGEN_CA_ID - session.add(first_mapped_variant) - session.commit() + # Seed mapped alleles with a ClinGen id on only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + clingen_allele_id=VALID_CLINGEN_CA_ID, + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3766,10 +3823,18 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, # The ClinVar control seeded in setup_router_db has db_version="11_2024", mapping to namespace clinvar.2024_11. clinvar_namespace = "clinvar.2024_11" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + # ClinVar control id=1 (db_version 11_2024) links to only the first variant's authoritative allele. + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3800,10 +3865,17 @@ def test_download_clinvar_namespace_with_no_matching_version( # clinvar.2023_01 does not match the seeded control (11_2024), so all rows should be NA. clinvar_namespace = "clinvar.2023_01" experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3830,10 +3902,17 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( matching_ns = "clinvar.2024_11" # matches db_version="11_2024" seeded in setup_router_db non_matching_ns = "clinvar.2023_01" # no controls with this version experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_clinvar_control_to_mapped_variant(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + clinvar_control_ids=[1], + annotate="first", + ) with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: published_score_set = publish_score_set(client, score_set["urn"]) @@ -3861,6 +3940,49 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( assert all(row[f"{non_matching_ns}.clinical_review_status"] == "NA" for row in rows) +def test_download_variant_data_as_of_reconstructs_annotation_layer( + session, data_provider, client, setup_router_db, data_files +): + """`as_of` time-travels the CSV annotation layer: an instant before the annotation was live yields NA, + while the current view yields the value. The resolved instant is echoed in X-As-Of.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + + # Seed the whole substrate live as of 2020 — live "now" but not at an as_of in 2000. + seeded_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g=TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION["hgvs_g"], + vep_consequence="missense_variant", + valid_from=seeded_at, + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + base_url = f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + + # Current: the annotation is live, and X-As-Of echoes "current". + current = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false"}) + assert current.status_code == 200 + assert current.headers["X-As-Of"] == "current" + current_rows = list(csv.DictReader(StringIO(current.text))) + assert any(row["vep.vep_functional_consequence"] == "missense_variant" for row in current_rows) + + # An instant before the annotation existed: nothing live -> every VEP value is NA, and X-As-Of echoes it. + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + before = client.get(base_url, params={"namespaces": "vep", "drop_na_columns": "false", "as_of": past.isoformat()}) + assert before.status_code == 200 + assert datetime.fromisoformat(before.headers["X-As-Of"]) == past + before_rows = list(csv.DictReader(StringIO(before.text))) + assert before_rows # variants (and their immutable scores) are still present + assert all(row["vep.vep_functional_consequence"] == "NA" for row in before_rows) + + def test_invalid_clinvar_namespace_returns_422(client, setup_router_db, data_files): """A clinvar namespace with an out-of-range month (13) is rejected with 422.""" experiment = create_experiment(client) @@ -4067,6 +4189,106 @@ def test_clinical_control_options_exclude_non_current(client, setup_router_db, s assert response.status_code == 404 +######################################################################################################################## +# Downloading a score set's variant details (VRS + Cat-VRS + annotations) +######################################################################################################################## + + +def test_get_score_set_variant_details_returns_ndjson(client, session, data_provider, data_files, setup_router_db): + """The whole-set variant-detail export streams NDJSON — one VariantDetail per mapped variant, each + carrying the flat preMapped/postMapped VRS pair and the spec-pure Cat-VRS.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urns = { + variant.urn + for variant in seed_annotation_substrate(session, score_set, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X) + } + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/x-ndjson") + assert response.headers["X-As-Of"] == "current" + assert response.headers["X-Total-Count"] == str(len(variant_urns)) + assert response.headers["X-Stream-Type"] == "variant-detail" + + records = parse_ndjson_response(response) + assert len(records) == len(variant_urns) + assert {record["urn"] for record in records} == variant_urns + for record in records: + # The flat VRS pair for VRS consumers, and the spec-pure Cat-VRS for the full picture. + assert record["preMapped"] == TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X + assert record["postMapped"] == TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + assert record.get("molecularRepresentation") is not None + + +def test_get_score_set_variant_details_omits_unmapped_variants( + client, session, data_provider, data_files, setup_router_db +): + """Only mapped variants carry VRS/detail, so an un-mapped variant is omitted — the record count is + the mapped subset, not the whole score set.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variants = seed_annotation_substrate(session, score_set, skip_first=True) + unmapped_urn = variants[0].urn # read before the request expires the instances + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + + records = parse_ndjson_response(response) + assert len(records) == score_set["numVariants"] - 1 + assert unmapped_urn not in {record["urn"] for record in records} + + +def test_get_score_set_variant_details_for_unmapped_returns_empty( + client, session, data_provider, data_files, setup_router_db +): + """A score set with no live mapping records has no annotatable variants — an empty NDJSON stream with + X-Total-Count 0, not a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # No new-substrate mapping records → no annotatable variants. + + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/variant-details") + assert response.status_code == 200 + assert response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(response) == [] + + +def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/variant-details") + assert response.status_code == 404 + + +def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): + """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate + as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — + ``as_of`` is a filter, so an empty match is an empty collection, never a 404.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + seed_annotation_substrate(session, score_set) + url = f"/api/v1/score-sets/{score_set['urn']}/variant-details" + + future = datetime(2999, 1, 1, tzinfo=timezone.utc) + future_response = client.get(url, params={"as_of": future.isoformat()}) + assert future_response.status_code == 200 + assert datetime.fromisoformat(future_response.headers["X-As-Of"]) == future + assert len(parse_ndjson_response(future_response)) == score_set["numVariants"] + + past = datetime(2000, 1, 1, tzinfo=timezone.utc) + past_response = client.get(url, params={"as_of": past.isoformat()}) + assert past_response.status_code == 200 + assert past_response.headers["X-Total-Count"] == "0" + assert parse_ndjson_response(past_response) == [] + + ######################################################################################################################## # Fetching annotated variants for a score set ######################################################################################################################## @@ -4605,19 +4827,32 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian def test_can_fetch_current_gnomad_variants_for_score_set(client, setup_router_db, session, data_provider, data_files): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + # gnomAD variant id=1 (seeded by setup_router_db) links to the first variant's authoritative allele. + variants = seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) + expected_urn = variants[0].urn + expected_digest = f"csv-auth-{variants[0].id}" response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants") assert response.status_code == 200 + assert response.headers["x-as-of"] == "current" response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 + assert variant_links[0]["variantUrn"] == expected_urn + assert variant_links[0]["alleleDigest"] == expected_digest gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4626,10 +4861,17 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version={TEST_GNOMAD_DATA_VERSION}") assert response.status_code == 200 @@ -4637,8 +4879,8 @@ def test_can_fetch_current_gnomad_variants_for_score_set_with_version( response_data = response.json() assert len(response_data) == 1 for gnomad_variant in response_data: - mapped_variants = gnomad_variant.pop("mappedVariants") - assert len(mapped_variants) == 1 + variant_links = gnomad_variant.pop("variantLinks") + assert len(variant_links) == 1 gnomad_variant_items = sorted(gnomad_variant.items()) assert gnomad_variant_items == sorted(TEST_SAVED_GNOMAD_VARIANT.items()) @@ -4647,10 +4889,17 @@ def test_cannot_fetch_current_gnomad_variants_for_score_set_with_nonexistent_ver client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn']}/gnomad-variants?version=nonexistent_version") assert response.status_code == 404 @@ -4667,10 +4916,17 @@ def test_cannot_fetch_gnomad_variants_for_nonexistent_score_set( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) - link_gnomad_variants_to_mapped_variants(session, score_set) + seed_csv_substrate( + session, + score_set, + assay_level="genomic", + hgvs_g="NC_000018.10:g.1A>G", + gnomad_variant_ids=[1], + annotate="first", + ) response = client.get(f"/api/v1/score-sets/{score_set['urn'] + 'xxx'}/gnomad-variants") @@ -4683,7 +4939,7 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( client, setup_router_db, session, data_provider, data_files ): experiment = create_experiment(client) - score_set = create_seq_score_set_with_mapped_variants( + score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" ) diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index a422bbe24..bb96bec08 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -140,11 +140,13 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["cdna-digest"]["level"] == "cdna" assert body["alleles"]["cdna-digest"]["hgvs"] == "NM_000546.6:c.1216G>A" assert body["alleles"]["cdna-digest"]["clingenAlleleId"] == "CA123" - assert "relation" not in body["alleles"]["cdna-digest"] # null relation dropped by exclude_none + assert ( + body["alleles"]["cdna-digest"]["relation"] is None + ) # measured allele: no relation to itself (null, not dropped) # The measured allele is the focus (isFocus); it carries no derivation (that axis describes the # *other* members relative to it), and pairs with its genomic projection sibling (projectionOf). assert body["alleles"]["cdna-digest"]["isFocus"] is True - assert "derivation" not in body["alleles"]["cdna-digest"] # null derivation dropped by exclude_none + assert body["alleles"]["cdna-digest"]["derivation"] is None # null, not dropped (exclude_none=False) assert body["alleles"]["cdna-digest"]["projectionOf"] == "gen-digest" assert body["alleles"]["gen-digest"]["isFocus"] is False assert body["alleles"]["gen-digest"]["relation"] == "coordinate_representation_of" @@ -153,16 +155,16 @@ def test_get_variant_detail_envelope(client, session, data_provider, data_files, assert body["alleles"]["prot-digest"]["level"] == "protein" assert body["alleles"]["prot-digest"]["hgvs"] == "NP_000537.3:p.Ala406Thr" assert body["alleles"]["prot-digest"]["relation"] == "translation_of" - # The apex is a deterministic projection here but pairs with nothing (projectionOf dropped as null). + # The apex is a deterministic projection here but pairs with nothing (projectionOf is null). assert body["alleles"]["prot-digest"]["derivation"] == "projection" - assert "projectionOf" not in body["alleles"]["prot-digest"] + assert body["alleles"]["prot-digest"]["projectionOf"] is None # The synonymous cousin (different projection group) surfaces as a member wearing co_encodes and is # labelled `convergent` (a distinct change sharing the consequence, not an ambiguous candidate). assert body["alleles"]["cousin-digest"]["relation"] == "co_encodes" assert body["alleles"]["cousin-digest"]["derivation"] == "convergent" assert body["annotations"]["cdna-digest"]["vep"]["consequence"] == "missense_variant" assert body["isCurrent"] is True - assert "supersededByScoreSet" not in body # dropped by exclude_none when current + assert body["supersededByScoreSet"] is None # null when current (stable envelope, not dropped) assert response.headers["X-As-Of"] == "current" @@ -182,8 +184,8 @@ def test_get_variant_detail_echoes_as_of_header(client, session, data_provider, assert historical.status_code == 200 assert historical.headers["X-As-Of"] == "2020-01-01T00:00:00+00:00" - # Before any mapping existed, the molecular layer is empty. - assert "molecularRepresentation" not in historical.json() + # Before any mapping existed, the molecular layer is empty — the stable envelope carries it as null. + assert historical.json()["molecularRepresentation"] is None def test_superseded_variant_self_describes(client, session, data_provider, data_files, setup_router_db): From cdbaeab715ea29184ef41ee80dbf95a70772579a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 22 Jul 2026 23:17:37 -0700 Subject: [PATCH 90/93] feat(score-sets): rewrite CSV export onto the allele substrate - replace the MappedVariant-keyed CSV joins with a substrate query (_csv_substrate_query) over the live mapping record, its authoritative allele, and the projection-group nucleotide sibling - add _gnomad_by_allele/_clinvar_by_allele batch fetches keyed by allele id, replacing the old mapped_variant_id-keyed lookups - reuse mapped_hgvs_by_level/get_protein_hgvs_by_record from score_set_variants so the CSV resolves post-mapped HGVS per level the same way the lean whole-set view does, instead of falling back to a per-row VRS parse - extract column planning (_plan_csv_columns) and header assembly (_csv_header_columns) out of get_score_set_variants_as_csv - thread as_of through the CSV export to time-travel the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) independently of the immutable scores/counts namespaces - add seed_csv_substrate test helper for the substrate-based fixtures --- src/mavedb/lib/score_sets.py | 528 ++++++++++++++++++++------------ tests/helpers/util/score_set.py | 119 ++++--- 2 files changed, 412 insertions(+), 235 deletions(-) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 3c1559510..e684efd7f 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -3,6 +3,7 @@ import logging import re from collections import Counter, defaultdict +from dataclasses import dataclass from datetime import datetime from operator import attrgetter from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence @@ -10,11 +11,12 @@ import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy import Integer, Select, and_, cast, func, or_, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.mave.constants import ( HGVS_NT_COLUMN, @@ -26,24 +28,25 @@ ) from mavedb.lib.mave.utils import is_csv_null from mavedb.lib.permissions import Action, has_permission +from mavedb.lib.score_set_variants import get_protein_hgvs_by_record, mapped_hgvs_by_level from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.allele import Allele from mavedb.models.clinical_control import ClinvarControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.clinvar_allele_link import ClinvarAlleleLink from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier from mavedb.models.ensembl_identifier import EnsemblIdentifier from mavedb.models.ensembl_offset import EnsemblOffset +from mavedb.models.enums.sequence_level import SequenceLevel from mavedb.models.experiment import Experiment from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.gnomad_allele_link import GnomadAlleleLink from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant from mavedb.models.mapping_record import MappingRecord from mavedb.models.mapping_record_allele import MappingRecordAllele from mavedb.models.publication_identifier import PublicationIdentifier @@ -61,6 +64,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant +from mavedb.models.vep_allele_consequence import VepAlleleConsequence from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: @@ -74,6 +78,9 @@ # e.g. "clinvar.2024_01" for January 2024. CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +# CSV rows come out in variant-number order (the integer after '#'); id breaks ties stably. +_VARIANT_NUMBER_ORDER = (cast(func.split_part(Variant.urn, "#", 2), Integer), Variant.id) + class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -606,6 +613,287 @@ def get_annotatable_variants( ) +@dataclass(frozen=True) +class _CsvMappedRow: + """The mapping-derived CSV fields for one variant, resolved from its live mapping record's + authoritative allele (post-mapped HGVS per level, assay-level HGVS, VRS digest, VEP consequence, + ClinGen id). gnomAD frequency and ClinVar assertions ride alongside as separate parallel lists + (they fan out per allele, not 1:1 like these).""" + + hgvs_g: Optional[str] + hgvs_c: Optional[str] + hgvs_p: Optional[str] + hgvs_assay_level: Optional[str] + vrs_digest: Optional[str] + vep_functional_consequence: Optional[str] + clingen_allele_id: Optional[str] + + +@dataclass(frozen=True) +class _CsvVariantData: + """The per-variant inputs to the CSV row builder, as positionally-aligned sequences (one entry per + variant, in output order). ``mappings`` / ``gnomad_data`` / ``clinvar_per_variant`` are ``None`` when the + requested namespaces don't need that dimension.""" + + variants: list[Variant] + mappings: Optional[list[Optional[_CsvMappedRow]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] + + +def _plan_csv_columns( + score_set: ScoreSet, + namespaces: List[str], + *, + include_custom_columns: Optional[bool], + include_post_mapped_hgvs: Optional[bool], +) -> tuple[dict[str, list[str]], dict[str, str]]: + """Plan the CSV's columns from the requested namespaces. Returns the ``namespace -> [column, ...]`` map + (in emission order) and the ``clinvar_namespace -> db_version`` map parsed out of any + ``clinvar.YEAR_MONTH`` namespaces (``db_version`` is stored on the control as ``MONTH_YEAR``).""" + assert type(score_set.dataset_columns) is dict + + columns: dict[str, list[str]] = { + "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], + "mavedb": [], + } + + if include_post_mapped_hgvs: + columns["mavedb"] += [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + for namespace in namespaces: + columns[namespace] = [] + + if include_custom_columns: + if "scores" in columns: + columns["scores"] = [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] + if "counts" in columns: + columns["counts"] = [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] + elif "scores" in columns: + columns["scores"].append(REQUIRED_SCORE_COLUMN) + + if "vep" in columns: + columns["vep"].append("vep_functional_consequence") + if "gnomad" in columns: + columns["gnomad"].append("gnomad_af") + if "clingen" in columns: + columns["clingen"].append("clingen_allele_id") + + clinvar_namespaces: dict[str, str] = {} + for cv_ns in namespaces: + m = CLINVAR_NS_PATTERN.match(cv_ns) + if m: + year, month = m.group(1), m.group(2) + clinvar_namespaces[cv_ns] = f"{month}_{year}" + columns[cv_ns] = ["clinical_significance", "clinical_review_status"] + + return columns, clinvar_namespaces + + +def _csv_header_columns(columns: dict[str, list[str]], *, namespaced: Optional[bool]) -> list[str]: + """Flatten the planned column map into the ordered CSV header. ClinVar-versioned namespaces are always + prefixed (to avoid collisions when multiple versions are requested); other non-``core`` namespaces are + prefixed only when ``namespaced`` is set; ``core`` is never prefixed.""" + header: list[str] = [] + for namespace, cols in columns.items(): + for col in cols: + if CLINVAR_NS_PATTERN.match(namespace) or (namespaced and namespace != "core"): + header.append(f"{namespace}.{col}") + else: + header.append(col) + + return header + + +def _csv_substrate_query(score_set_id: int, *, as_of: Optional[datetime]) -> Select[Any]: + """The base per-variant query for the annotation namespaces: each variant LEFT-joined to its live mapping + record, its authoritative (measured) allele (digest / ClinGen id / VEP consequence), and the + projection-group nucleotide sibling — mirroring the lean whole-set view's join + (:func:`mavedb.lib.score_set_variants.get_lean_score_set_variants`). LEFT joins with ``live_at`` in the ON + clause keep unmapped variants (with null mapped fields); a WHERE would collapse them out. The protein slot + and the gnomAD/ClinVar dimensions are resolved by separate fetches and stitched on by id.""" + sibling_link = aliased(MappingRecordAllele) + sibling_allele = aliased(Allele) + return ( + select( + Variant, + MappingRecord.id.label("mapping_record_id"), + MappingRecord.assay_level, + MappingRecord.hgvs_assay_level, + Allele.id.label("allele_id"), + Allele.vrs_digest, + Allele.clingen_allele_id, + VepAlleleConsequence.functional_consequence, + sibling_allele.level.label("sibling_level"), + # Exactly one of the sibling's hgvs_g/hgvs_c is populated (it is a nucleotide allele), so + # coalesce yields its canonical string regardless of which nucleotide level it sits at. + func.coalesce(sibling_allele.hgvs_g, sibling_allele.hgvs_c).label("sibling_hgvs"), + ) + .outerjoin(MappingRecord, and_(MappingRecord.variant_id == Variant.id, MappingRecord.live_at(as_of))) + .outerjoin( + MappingRecordAllele, + and_( + MappingRecordAllele.mapping_record_id == MappingRecord.id, + MappingRecordAllele.is_authoritative.is_(True), + MappingRecordAllele.live_at(as_of), + ), + ) + .outerjoin(Allele, Allele.id == MappingRecordAllele.allele_id) + .outerjoin( + VepAlleleConsequence, + and_(VepAlleleConsequence.allele_id == Allele.id, VepAlleleConsequence.live_at(as_of)), + ) + .outerjoin( + sibling_link, + and_( + sibling_link.mapping_record_id == MappingRecord.id, + sibling_link.projection_group == MappingRecordAllele.projection_group, + sibling_link.is_authoritative.is_(False), + sibling_link.live_at(as_of), + ), + ) + .outerjoin(sibling_allele, sibling_allele.id == sibling_link.allele_id) + .where(Variant.score_set_id == score_set_id) + .order_by(*_VARIANT_NUMBER_ORDER) + ) + + +def _apply_pagination(query: Select[Any], start: Optional[int], limit: Optional[int]) -> Select[Any]: + """Apply the shared ``start``/``limit`` window (offset/limit) to a variant query. A falsy ``start`` or + ``limit`` leaves that bound off.""" + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + return query + + +def _gnomad_by_allele(db: Session, allele_ids: list[int], *, as_of: Optional[datetime]) -> dict[int, GnomADVariant]: + """The live gnomAD frequency record at the currently-served release (:data:`GNOMAD_DATA_VERSION`) for each + of ``allele_ids``, keyed by allele id — at most one per allele (single-live link, narrowed to the release + the CSV's ``gnomad_af`` column reports).""" + if not allele_ids: + return {} + + rows = db.execute( + select(GnomadAlleleLink.allele_id, GnomADVariant) + .join(GnomADVariant, GnomADVariant.id == GnomadAlleleLink.gnomad_variant_id) + .where(GnomadAlleleLink.allele_id.in_(allele_ids)) + .where(GnomadAlleleLink.live_at(as_of)) + .where(GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION) + ) + return {allele_id: gnomad_variant for allele_id, gnomad_variant in rows.tuples()} + + +def _clinvar_by_allele( + db: Session, allele_ids: list[int], clinvar_namespaces: dict[str, str], *, as_of: Optional[datetime] +) -> dict[str, dict[int, ClinvarControl]]: + """For each requested ClinVar namespace, the live control at that release for each of ``allele_ids`` + (``allele id -> control``). ClinVar links are multi-live (one per release), so each namespace's query is + narrowed to its own ``db_version``.""" + by_namespace: dict[str, dict[int, ClinvarControl]] = {} + for ns, db_version in clinvar_namespaces.items(): + per_allele: dict[int, ClinvarControl] = {} + if allele_ids: + rows = db.execute( + select(ClinvarAlleleLink.allele_id, ClinvarControl) + .join(ClinvarControl, ClinvarControl.id == ClinvarAlleleLink.clinvar_control_id) + .where(ClinvarAlleleLink.allele_id.in_(allele_ids)) + .where(ClinvarAlleleLink.live_at(as_of)) + .where(ClinvarControl.db_name == "ClinVar", ClinvarControl.db_version == db_version) + ) + per_allele = {allele_id: control for allele_id, control in rows.tuples()} + + by_namespace[ns] = per_allele + + return by_namespace + + +def _csv_mapped_row(row: Any, protein_hgvs_by_record: dict[int, str]) -> _CsvMappedRow: + """Assemble one variant's :class:`_CsvMappedRow` from a substrate-query row: the canonical post-mapped + HGVS at each level (reconstructed via the shared projection rule) plus the assay-level string, VRS digest, + VEP consequence, and ClinGen id off the authoritative allele.""" + slots = mapped_hgvs_by_level( + assay_level=SequenceLevel(row.assay_level) if row.assay_level else None, + assay_level_hgvs=row.hgvs_assay_level, + sibling_level=row.sibling_level, + sibling_hgvs=row.sibling_hgvs, + protein_hgvs=protein_hgvs_by_record.get(row.mapping_record_id), + ) + return _CsvMappedRow( + hgvs_g=slots.get(SequenceLevel.genomic.value), + hgvs_c=slots.get(SequenceLevel.cdna.value), + hgvs_p=slots.get(SequenceLevel.protein.value), + hgvs_assay_level=row.hgvs_assay_level, + vrs_digest=row.vrs_digest, + vep_functional_consequence=row.functional_consequence, + clingen_allele_id=row.clingen_allele_id, + ) + + +def _fetch_csv_variant_data( + db: Session, + score_set: ScoreSet, + *, + need_mappings: bool, + need_gnomad: bool, + clinvar_namespaces: dict[str, str], + start: Optional[int], + limit: Optional[int], + as_of: Optional[datetime], +) -> _CsvVariantData: + """Fetch the paginated per-variant CSV inputs. When no annotation namespace is requested this is a plain + scan of the score set's variants (no substrate join, ``as_of`` irrelevant); otherwise it joins the live + mapping substrate and batch-fetches the gnomAD/ClinVar dimensions keyed by authoritative allele id.""" + if not need_mappings: + # Fast path: scores/counts (and bare core) are a straight paginated scan in variant-number order, + # untouched by as_of (it only reconstructs the — absent here — annotation layer). + query = _apply_pagination( + select(Variant).where(Variant.score_set_id == score_set.id).order_by(*_VARIANT_NUMBER_ORDER), + start, + limit, + ) + return _CsvVariantData(list(db.scalars(query).all()), None, None, None) + + result = db.execute(_apply_pagination(_csv_substrate_query(score_set.id, as_of=as_of), start, limit)).all() + + # Protein slot: the standalone protein subquery over the whole set, stitched back by record id. + protein_hgvs_by_record = get_protein_hgvs_by_record(db, score_set.id, as_of=as_of) + # The authoritative allele ids on this page — the join key for the batch gnomAD/ClinVar fetches. + allele_ids = [row.allele_id for row in result if row.allele_id is not None] + gnomad_by_allele = _gnomad_by_allele(db, allele_ids, as_of=as_of) if need_gnomad else {} + clinvar_by_allele = _clinvar_by_allele(db, allele_ids, clinvar_namespaces, as_of=as_of) + + variants: list[Variant] = [] + mappings: list[Optional[_CsvMappedRow]] = [] + gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = ( + [] if clinvar_namespaces else None + ) + for row in result: + variants.append(row.Variant) + allele_id = row.allele_id + mappings.append(_csv_mapped_row(row, protein_hgvs_by_record) if row.mapping_record_id is not None else None) + if gnomad_data is not None: + gnomad_data.append(gnomad_by_allele.get(allele_id) if allele_id is not None else None) + if clinvar_per_variant is not None: + clinvar_per_variant.append( + { + ns: (clinvar_by_allele[ns].get(allele_id) if allele_id is not None else None) + for ns in clinvar_namespaces + } + ) + + return _CsvVariantData(variants, mappings, gnomad_data, clinvar_per_variant) + + def get_score_set_variants_as_csv( db: Session, score_set: ScoreSet, @@ -616,6 +904,7 @@ def get_score_set_variants_as_csv( drop_na_columns: Optional[bool] = None, include_custom_columns: Optional[bool] = True, include_post_mapped_hgvs: Optional[bool] = False, + as_of: Optional[datetime] = None, ) -> str: """ Get the variant data from a score set as a CSV string. @@ -643,180 +932,47 @@ def get_score_set_variants_as_csv( include_post_mapped_hgvs : bool, optional Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. + as_of : datetime, optional + Reconstruct the annotation layer (post-mapped HGVS, VEP, gnomAD, ClinVar) as it stood at this + instant, over the variant's immutable submitted HGVS/scores/counts. Defaults to the currently-live + rows. Has no effect on the ``scores``/``counts`` namespaces, which are as-of-invariant. Returns _______ str The CSV string containing the variant data. """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) + columns, clinvar_namespaces = _plan_csv_columns( + score_set, + namespaces, + include_custom_columns=include_custom_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + ) + need_mappings = bool( + include_post_mapped_hgvs or clinvar_namespaces or ({"clingen", "vep", "gnomad"} & set(namespaces)) ) need_gnomad = "gnomad" in namespaces - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) + data = _fetch_csv_variant_data( + db, + score_set, + need_mappings=need_mappings, + need_gnomad=need_gnomad, + clinvar_namespaces=clinvar_namespaces, + start=start, + limit=limit, + as_of=as_of, ) - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinvarControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinvarControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinvarControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinvarControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinvarControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinvarControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, + data.variants, + columns=columns, namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) + mappings=data.mappings, + gnomad_data=data.gnomad_data, + clinvar_data_by_ns=data.clinvar_per_variant, + ) + rows_columns = _csv_header_columns(columns, namespaced=namespaced) if drop_na_columns: rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) @@ -874,7 +1030,7 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: def variant_to_csv_row( variant: Variant, columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, + mapping: Optional["_CsvMappedRow"] = None, gnomad_data: Optional[GnomADVariant] = None, clinvar_data_by_ns: Optional[dict[str, Optional[ClinvarControl]]] = None, namespaced: Optional[bool] = None, @@ -891,8 +1047,8 @@ def variant_to_csv_row( Columns to serialize. namespaced: Optional[bool] = None Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. + mapping : _CsvMappedRow, optional + The variant's mapping-derived fields, resolved from its live mapping record's authoritative allele. gnomad_data : variant.models.GnomADVariant, optional gnomAD variant data corresponding to the variant. clinvar_data_by_ns : dict[str, Optional[ClinvarControl]], optional @@ -921,39 +1077,19 @@ def variant_to_csv_row( # export columns in the `core` namespace without a namespace row[column_key] = value for column_key in columns.get("mavedb", []): + # The canonical post-mapped HGVS at each level (and the assay-level string / VRS digest) are + # pre-resolved onto `mapping` from the authoritative allele + its projection group; no VRS-object + # fallback parse is needed here — the substrate stores the canonical string per level directly. if column_key == "post_mapped_hgvs_g": value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) - if mapping and mapping.post_mapped - else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - elif column_key == "post_mapped_hgvs_p": value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped, combine_cis=True) - if mapping and mapping.post_mapped - else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - elif column_key == "post_mapped_hgvs_c": value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep elif column_key == "post_mapped_hgvs_at_assay_level": value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep + value = mapping.vrs_digest if mapping and mapping.vrs_digest else na_rep if is_null(value): value = na_rep key = f"mavedb.{column_key}" if namespaced else column_key @@ -1022,7 +1158,7 @@ def variant_to_csv_row( def variants_to_csv_rows( variants: Sequence[Variant], columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + mappings: Optional[Sequence[Optional["_CsvMappedRow"]]] = None, gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinvarControl]]]]] = None, namespaced: Optional[bool] = None, @@ -1039,8 +1175,8 @@ def variants_to_csv_rows( Columns to serialize. namespaced: Optional[bool] = None Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. + mappings : list[Optional[_CsvMappedRow]], optional + Per-variant mapping-derived fields (parallel to ``variants``). gnomad_data : list[Optional[variant.models.GnomADVariant]], optional List of gnomAD variant data corresponding to the variants. clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinvarControl]]]], optional @@ -1053,7 +1189,7 @@ def variants_to_csv_rows( list[dict[str, Any]] """ n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _mappings: Sequence[Optional["_CsvMappedRow"]] = mappings if mappings is not None else [None] * n _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n _clinvar: Sequence[Optional[dict[str, Optional[ClinvarControl]]]] = ( clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n diff --git a/tests/helpers/util/score_set.py b/tests/helpers/util/score_set.py index 7e9022db7..35fd9e382 100644 --- a/tests/helpers/util/score_set.py +++ b/tests/helpers/util/score_set.py @@ -8,8 +8,6 @@ from fastapi.testclient import TestClient from sqlalchemy import select -from mavedb.models.clinical_control import ClinvarControl as ClinicalControlDbModel -from mavedb.models.gnomad_variant import GnomADVariant as GnomADVariantDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel @@ -205,7 +203,7 @@ def create_acc_score_set_with_variants( return score_set -def seed_annotation_substrate(db, score_set, *, skip_first=False): +def seed_annotation_substrate(db, score_set, *, skip_first=False, pre_mapped=None): """Give a score set's variants a live ``MappingRecord`` with one authoritative allele. The VA endpoints build a ``VariantAnnotationContext`` from the ``MappingRecord`` / ``Allele`` @@ -214,7 +212,8 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): mirrors what the mapping pipeline writes. (Global seeding from ``mock_worker_vrs_mapping`` is deferred to Slice 5.2/5.3, where it can be reconciled with the clinical-controls seeding.) - With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the + ``pre_mapped`` sets the assayed-level VRS on each record (the flat ``preMapped`` on the variant + detail). With ``skip_first=True`` the first variant is left un-mapped on the new substrate (for the "some variants were not mapped" cases). Returns the score set's variants in id order. """ variants = db.scalars( @@ -227,6 +226,7 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): seed_mapping_record( db, variant, + pre_mapped=pre_mapped, alleles=[ AlleleSpec( digest=f"va-allele-{variant.id}", @@ -239,6 +239,82 @@ def seed_annotation_substrate(db, score_set, *, skip_first=False): return variants +def seed_csv_substrate( + db, + score_set, + *, + assay_level="genomic", + hgvs_g=None, + hgvs_c=None, + hgvs_p=None, + vep_consequence=None, + clingen_allele_id=None, + gnomad_variant_ids=(), + clinvar_control_ids=(), + annotate="all", + valid_from=None, +): + """Seed every variant of a score set with a live mapping record on the allele substrate, shaped for the + CSV export reader (``get_score_set_variants_as_csv``). + + The authoritative allele sits at ``assay_level`` and carries the annotations the ``vep``/``clingen``/ + ``gnomad``/``clinvar`` namespaces read (VEP consequence, ClinGen id, gnomAD/ClinVar links). For a + nucleotide assay, a projection-group sibling at the other nucleotide level and a protein apex allele are + added so the ``post_mapped_hgvs_{g,c,p}`` triple reconstructs the same way the lean view does. ``hgvs_g``/ + ``hgvs_c``/``hgvs_p`` supply the canonical string at each level (the one at ``assay_level`` is also the + record's ``hgvs_assay_level``). ``annotate`` controls which variants get the authoritative-allele + annotations: ``"all"`` (every variant) or ``"first"`` (only the first, for "one linked control" cases). + """ + variants = db.scalars( + select(VariantDbModel) + .join(ScoreSetDbModel) + .where(ScoreSetDbModel.urn == score_set["urn"]) + .order_by(VariantDbModel.id) + ).all() + level_hgvs = {"genomic": hgvs_g, "cdna": hgvs_c, "protein": hgvs_p} + assay_hgvs = level_hgvs[assay_level] + + for index, variant in enumerate(variants): + annotated = annotate == "all" or index == 0 + specs = [ + AlleleSpec( + digest=f"csv-auth-{variant.id}", + level=assay_level, + is_authoritative=True, + projection_group=0, + hgvs_g=hgvs_g if assay_level == "genomic" else None, + hgvs_c=hgvs_c if assay_level == "cdna" else None, + hgvs_p=hgvs_p if assay_level == "protein" else None, + vep_consequence=vep_consequence if annotated else None, + clingen_allele_id=clingen_allele_id if annotated else None, + gnomad_variant_ids=tuple(gnomad_variant_ids) if annotated else (), + clinvar_control_ids=tuple(clinvar_control_ids) if annotated else (), + ) + ] + # Nucleotide assay: add the other-nucleotide projection sibling and the protein apex so the + # g/c/p triple reconstructs. A protein assay populates only the protein slot (no c/g fan-out). + if assay_level in ("genomic", "cdna"): + other_level = "cdna" if assay_level == "genomic" else "genomic" + if level_hgvs[other_level] is not None: + specs.append( + AlleleSpec( + digest=f"csv-sib-{variant.id}", + level=other_level, + projection_group=0, + hgvs_g=hgvs_g if other_level == "genomic" else None, + hgvs_c=hgvs_c if other_level == "cdna" else None, + ) + ) + if hgvs_p is not None: + specs.append(AlleleSpec(digest=f"csv-prot-{variant.id}", level="protein", hgvs_p=hgvs_p)) + + seed_mapping_record( + db, variant, assay_level=assay_level, hgvs_assay_level=assay_hgvs, alleles=specs, valid_from=valid_from + ) + + return variants + + def link_clinical_controls_to_alleles(db, score_set): """Seed the new-model annotation graph for a score set's first two variants and link the two seeded ClinVar controls to their alleles via ``ClinvarAlleleLink``. @@ -265,41 +341,6 @@ def link_clinical_controls_to_alleles(db, score_set): ) -def link_clinvar_control_to_mapped_variant(db, score_set): - """Link the seeded ClinVar clinical control (id=1) to the first mapped variant of a score set.""" - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - - mapped_variants[0].clinical_controls.append( - db.scalar(select(ClinicalControlDbModel).where(ClinicalControlDbModel.id == 1)) - ) - - db.add(mapped_variants[0]) - db.commit() - - -def link_gnomad_variants_to_mapped_variants(db, score_set): - mapped_variants = db.scalars( - select(MappedVariantDbModel) - .join(VariantDbModel) - .join(ScoreSetDbModel) - .where(ScoreSetDbModel.urn == score_set["urn"]) - ).all() - - # The first mapped variant gets the gnomAD variant. - mapped_variants[0].gnomad_variants.append( - db.scalar(select(GnomADVariantDbModel).where(GnomADVariantDbModel.id == 1)) - ) - - db.add(mapped_variants[0]) - db.add(mapped_variants[1]) - db.commit() - - def mock_worker_vrs_mapping(client, db, score_set, alleles=True): # The mapping job is tested elsewhere, so insert mapped variants manually. variants = db.scalars( From d53c273f316a4d19a9a14a098c711e88519a3417 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 23 Jul 2026 11:07:51 -0700 Subject: [PATCH 91/93] feat(variants): add migration stubs for retired mapped-variant routes The mapped-variant endpoints removed earlier on this branch are still live on main, so merging is what actually retires them for public API consumers. Soften that landing per-route, based on whether the replacement is wire-compatible: - GET /score-sets/{urn}/mapped-variants now returns 410 Gone pointing at variant-details, since the streaming NDJSON shape isn't compatible with the old JSON array and a redirect would mislead callers - /mapped-variants/{urn} and its va/* and vrs/{identifier} siblings 301-redirect onto their /variants equivalents, since that move was a straight relocation with an unchanged response shape - POST /variants/clingen-allele-id-lookups stays fully removed with no stub; it was effectively an internal-only route - add a reusable 410 entry to routers/shared.py's BASE_RESPONSES --- src/mavedb/routers/mapped_variant.py | 96 ++++++++++++++++++++++++++++ src/mavedb/routers/score_sets.py | 25 ++++++++ src/mavedb/routers/shared.py | 2 + src/mavedb/server_main.py | 2 + tests/routers/test_mapped_variant.py | 53 +++++++++++++++ tests/routers/test_score_set.py | 13 ++++ 6 files changed, 191 insertions(+) create mode 100644 src/mavedb/routers/mapped_variant.py create mode 100644 tests/routers/test_mapped_variant.py diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py new file mode 100644 index 000000000..a42fa2503 --- /dev/null +++ b/src/mavedb/routers/mapped_variant.py @@ -0,0 +1,96 @@ +import logging +from typing import Annotated +from urllib.parse import quote + +from fastapi import APIRouter, Path, Request +from fastapi.responses import RedirectResponse +from ga4gh.core.identifiers import GA4GH_IR_REGEXP + +from mavedb.lib.logging import LoggedRoute +from mavedb.routers.shared import ROUTER_BASE_PREFIX + +TAG_NAME = "Variants" + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix=f"{ROUTER_BASE_PREFIX}/mapped-variants", + tags=[TAG_NAME], + route_class=LoggedRoute, +) + + +def _redirect(request: Request, target: str) -> RedirectResponse: + url = f"{target}?{request.url.query}" if request.url.query else target + return RedirectResponse(url=url, status_code=301) + + +@router.get( + "/{urn}", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}", +) +def redirect_mapped_variant(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}") + + +@router.get( + "/{urn}/va/study-result", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/study-result", +) +def redirect_mapped_variant_study_result(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/study-result`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/study-result") + + +@router.get( + "/{urn}/va/functional-statement", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/functional-statement", +) +def redirect_mapped_variant_functional_impact_statement(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/functional-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/functional-statement") + + +@router.get( + "/{urn}/va/pathogenicity-statement", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/{urn}/va/pathogenicity-statement", +) +def redirect_mapped_variant_acmg_evidence_line(*, urn: str, request: Request) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/{urn}/va/pathogenicity-statement`` instead.""" + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/{quote(urn, safe='')}/va/pathogenicity-statement") + + +@router.get( + "/vrs/{identifier}", + status_code=301, + deprecated=True, + summary="Moved to GET /variants/vrs/{identifier}", +) +def redirect_mapped_variants_by_identifier( + *, + identifier: Annotated[ + str, + Path( + description="String, a valid GA4GH digest based identifier.", + json_schema_extra={"example": "ga4gh:SQ.0123abcd"}, + regex=GA4GH_IR_REGEXP, + ), + ], + request: Request, +) -> RedirectResponse: + """This resource has moved. Use ``GET /variants/vrs/{identifier}`` instead. + + Note that the replacement's ``only_current`` boolean query parameter has been superseded by + ``as_of``; a caller relying on ``only_current=false`` should switch to passing an explicit + ``as_of`` timestamp rather than expecting it to carry over through this redirect. + """ + return _redirect(request, f"{ROUTER_BASE_PREFIX}/variants/vrs/{quote(identifier, safe='')}") diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 6bbed79f8..9b3203726 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -88,6 +88,7 @@ ACCESS_CONTROL_ERROR_RESPONSES, BASE_400_RESPONSE, BASE_409_RESPONSE, + BASE_RESPONSES, GATEWAY_ERROR_RESPONSES, PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, @@ -972,6 +973,30 @@ async def get_score_set_variant_details( ) +@router.get( + "/score-sets/{urn}/mapped-variants", + status_code=410, + deprecated=True, + responses={410: BASE_RESPONSES[410]}, + summary="Removed; see GET /score-sets/{urn}/variant-details", +) +def get_score_set_mapped_variants_removed(*, urn: str) -> Any: + """This endpoint has been permanently removed. + + Its JSON-array response has been replaced by a streaming NDJSON payload with a different + field shape (flat ``preMapped``/``postMapped`` VRS pair rather than a ``MappedVariant``-keyed + record), so the two are not wire-compatible and this route does not redirect. Use + ``GET /score-sets/{urn}/variant-details`` instead. + """ + raise HTTPException( + status_code=410, + detail=( + f"GET /score-sets/{urn}/mapped-variants has been removed. " + f"Use GET /score-sets/{urn}/variant-details instead." + ), + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, diff --git a/src/mavedb/routers/shared.py b/src/mavedb/routers/shared.py index f98edb39c..2cddfd23d 100644 --- a/src/mavedb/routers/shared.py +++ b/src/mavedb/routers/shared.py @@ -8,6 +8,7 @@ 403: {"description": "Forbidden. Insufficient permissions."}, 404: {"description": "Resource not found."}, 409: {"description": "Conflict with current resource state."}, + 410: {"description": "Gone. This resource has been permanently removed."}, 416: {"description": "Requested range not satisfiable."}, 422: {"description": "Unprocessable entity. Validation failed."}, 429: {"description": "Too many requests. Rate limit exceeded."}, @@ -23,6 +24,7 @@ BASE_403_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {403: BASE_RESPONSES[403]} BASE_404_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {404: BASE_RESPONSES[404]} BASE_409_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {409: BASE_RESPONSES[409]} +BASE_410_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {410: BASE_RESPONSES[410]} BASE_416_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {416: BASE_RESPONSES[416]} BASE_422_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {422: BASE_RESPONSES[422]} BASE_429_RESPONSE: Mapping[Union[int, str], dict[str, Any]] = {429: BASE_RESPONSES[429]} diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index d3e8c4221..7109cedff 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -52,6 +52,7 @@ hgvs, job_runs, licenses, + mapped_variant, orcid, permissions, pipelines, @@ -106,6 +107,7 @@ app.include_router(job_runs.router) app.include_router(licenses.router) # app.include_router(log.router) +app.include_router(mapped_variant.router) app.include_router(orcid.router) app.include_router(permissions.router) app.include_router(pipelines.router) diff --git a/tests/routers/test_mapped_variant.py b/tests/routers/test_mapped_variant.py new file mode 100644 index 000000000..e118ba192 --- /dev/null +++ b/tests/routers/test_mapped_variant.py @@ -0,0 +1,53 @@ +# ruff: noqa: E402 +"""Router tests for the retired ``/mapped-variants`` surface. + +The router itself is gone (see ``tests/routers/test_variant_annotation.py``); what remains at this +prefix is a set of 301 redirects onto the routes' new home under ``/variants``, kept so external +callers hitting the old public paths get pointed at the replacement instead of a bare 404. +""" + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from urllib.parse import quote, quote_plus + + +TEST_URN = "urn:mavedb:00000001-a-1#1" + + +def test_redirect_mapped_variant(client): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}" + + +@pytest.mark.parametrize( + "suffix", + ["va/study-result", "va/functional-statement", "va/pathogenicity-statement"], +) +def test_redirect_mapped_variant_va_routes(client, suffix): + response = client.get(f"/api/v1/mapped-variants/{quote_plus(TEST_URN)}/{suffix}", follow_redirects=False) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/{quote_plus(TEST_URN)}/{suffix}" + + +def test_redirect_mapped_variants_by_identifier(client): + identifier = "ga4gh:VA.0123456789abcdefghijklmnopqrstuv" + response = client.get( + f"/api/v1/mapped-variants/vrs/{identifier}?only_current=false", + follow_redirects=False, + ) + + assert response.status_code == 301 + assert response.headers["location"] == f"/api/v1/variants/vrs/{quote(identifier, safe='')}?only_current=false" + + +def test_redirect_mapped_variants_by_identifier_rejects_malformed_identifier(client): + response = client.get("/api/v1/mapped-variants/vrs/not-a-valid-identifier", follow_redirects=False) + + assert response.status_code == 422 diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 2bc830db3..4ade659ba 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -4265,6 +4265,19 @@ def test_cannot_get_variant_details_for_nonexistent_score_set(client, setup_rout assert response.status_code == 404 +def test_score_set_mapped_variants_is_permanently_removed(client): + """The old JSON-array ``mapped-variants`` route is gone in favor of the streaming NDJSON + ``variant-details`` route; it returns 410 rather than redirecting since the two are not + wire-compatible.""" + urn = "urn:mavedb:00000001-a-1" + response = client.get(f"/api/v1/score-sets/{urn}/mapped-variants") + + assert response.status_code == 410 + assert response.json()["detail"] == ( + f"GET /score-sets/{urn}/mapped-variants has been removed. Use GET /score-sets/{urn}/variant-details instead." + ) + + def test_get_score_set_variant_details_honors_as_of(client, session, data_provider, data_files, setup_router_db): """``as_of`` time-travels the molecular layer: a far-future instant sees the freshly-seeded substrate as live (the full set), a far-past instant sees no live mapping (an empty stream). Either way 200 — From fbfd4b91d34e6d54179faa10b5c4057321beacfc Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 23 Jul 2026 22:23:41 -0700 Subject: [PATCH 92/93] fix(measurements): reach a CA's protein consequence unconditionally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The include_nucleotide_siblings flag never actually gated siblings — reverse translation cross-links every record to its full synonymous nt set, so a sibling's record already links the anchor allele. Remove the flag (api + query param) and make the protein-apex fold-in unconditional for a CA query, so one behavior serves both the variant page and search: a CA reaches its protein consequence even when that protein assay hasn't been reverse-translated (its record links only the protein node). Resolve the apex once in _resolve_protein_apex, and instrument it: warn on a divergent apex (>1 PAID), and publish apex PAID/unregistered counts and the protein-consequence / apex-only-unresolved rates to the request log. Update tests to the one-behavior model. --- src/mavedb/lib/allele_measurements.py | 229 ++++++++++++++++---------- src/mavedb/routers/clingen_alleles.py | 12 -- tests/lib/test_allele_measurements.py | 182 +++++++++++++++----- tests/routers/test_clingen_allele.py | 22 +-- 4 files changed, 298 insertions(+), 147 deletions(-) diff --git a/src/mavedb/lib/allele_measurements.py b/src/mavedb/lib/allele_measurements.py index 6ca2a6d23..6cfb12797 100644 --- a/src/mavedb/lib/allele_measurements.py +++ b/src/mavedb/lib/allele_measurements.py @@ -1,29 +1,36 @@ -"""The ClinGen-allele-centric variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. - -Given a ClinGen allele id (a nucleotide ``CA`` or a protein ``PA``), list every measurement whose -**cross-layer equivalence class** touches that allele — not just the exact allele. The equivalence -relation is co-membership in a ``MappingRecord``'s allele set (the mapping + reverse-translation graph, -:mod:`lib.alleles`): a ``CA`` anchors on its nucleotide alleles (genomic + coding share the CA), so the -list gathers the nt measurements of that change (**direct**) *and* the protein measurements of its -consequence (**related**); a ``PA`` anchors on the protein allele, gathering the protein measurements of -that change (**direct**) *and* every nt measurement that encodes it (**related**). The asymmetry falls -out of *which* allele anchors the co-membership — a sibling nt change that also encodes the same protein -is not pulled onto a CA page, because its record links the protein but not the anchor nt allele. -``include_nucleotide_siblings`` opts out of that asymmetry for discovery surfaces — see -:func:`get_allele_measurements`. - -Distinct from :func:`lib.variant_detail.get_variant_detail`, which is the *record-scoped*, all-levels -detail of one *selected* measurement. This list is the *cross-record* union — who else measured -something in the equivalence class. - -Authorization uses ``has_permission`` directly (matching ``lib/score_sets.py``): ``user_data`` is checked -per entity — score-set READ gates whether a measurement appears at all (a private score set's measurement -never leaks), and calibration READ gates the inline classification (withheld while the measurement still -shows). The two-gate split is deliberate: a readable measurement can carry an unreadable classification. -``as_of`` reconstructs the molecular layer (which records/links are live) at a past instant; scores are -immutable and calibrations carry no valid-time, so both are as-of-invariant. +"""The variant page's measurements list — ``GET /clingen-alleles/{caid}/measurements``. + +Given a ClinGen allele id — a nucleotide ``CA`` or a protein ``PA`` — return every measurement related to +it across sequence levels. Two measurements are related when their mapping records share an allele (in the +mapping + reverse-translation graph). A ``CA`` anchors on its nt alleles; a ``PA`` on the protein allele. +Each measurement is labelled by its own measured level relative to the query: ``direct`` (measured at the +query), ``protein_consequence``, or ``nucleotide_encoding``. + +Example. Reference codon ``TTT`` (Phe200): both ``c.598T>C`` and the sibling ``c.600T>A`` encode +``p.Phe200Leu``. Call them ``CA1``, ``CA2``, ``PA1`` — X assays ``c.598T>C`` (nt), Y assays +``p.Phe200Leu`` (protein), Z assays ``c.600T>A`` (nt): + + query CA1 → X direct, Y protein_consequence, Z nucleotide_encoding + query PA1 → Y direct, X and Z nucleotide_encoding + +A CA query reaches its protein consequence (Y) through the shared protein node ``PA1``. This is what makes +Y reachable when a protein assay has not been reverse-translated: its record then links only the protein +node, with no nt allele for the CA anchor to match, so the apex is the only path to it. The sibling Z is +reached without the apex — reverse translation links every record to its full synonymous nt set, so Z's +record already links ``CA1`` directly. There is no page/search distinction: every surface runs one query. + +The protein-consequence step is resolved once, in :func:`_resolve_protein_apex`, which also measures it: +a consequence resolving to more than one protein id (a data-integrity signal) is logged, and the number of +consequences reached with no reverse-translation data of their own is written to the request log. + +Authorization (``has_permission``, per ``lib/score_sets.py``): score-set READ gates whether a measurement +appears; calibration READ gates only its inline classification. ``as_of`` reconstructs which records and +links were live at a past instant; scores and calibrations are as-of-invariant. + +Distinct from :func:`lib.variant_detail.get_variant_detail` — the record-scoped detail of one measurement. """ +import logging from dataclasses import dataclass from datetime import date, datetime from enum import Enum @@ -32,6 +39,7 @@ from sqlalchemy import and_, select from sqlalchemy.orm import Session, aliased, joinedload +from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, has_permission from mavedb.lib.score_calibrations import calibration_preference_key, classification_evidence_strength from mavedb.lib.types.authentication import UserData @@ -47,31 +55,25 @@ ) from mavedb.models.variant import Variant +logger = logging.getLogger(__name__) + class MeasurementRelationship(str, Enum): - """The relationship of a measurement to the queried ClinGen allele. ``direct`` = the measurement was - assayed *at* this allele; the other two are the RT-related measurements, named for how they relate - to the query. - """ + """How a measurement relates to the queried ClinGen id, by the measurement's *measured* level.""" - # the measurement was assayed *at* this allele - direct = "direct" - # CA query, protein measurement of consequence(N) - protein_consequence = "protein_consequence" - # A nt measurement encoding the queried change's protein consequence: a PA query's encodings of P, or — - # only under ``include_nucleotide_siblings`` — a CA query's sibling nt changes that share consequence(N). + direct = "direct" # assayed at the queried allele + protein_consequence = "protein_consequence" # protein measurement of a CA's consequence + # an nt measurement of a PA's encoding, or a CA's sibling nt change (encodes the same protein) nucleotide_encoding = "nucleotide_encoding" @dataclass(frozen=True) class AlleleMeasurement: - """One measurement in the queried allele's equivalence class (transit; serialized by - ``view_models.allele_measurement``). + """One measurement in the query's equivalence class (transit; serialized by ``view_models``). - ``assay_level`` is the level at which *this* measurement was assayed (protein/cdna/genomic) — the - clinically load-bearing fact, always shown. ``relationship`` says how it relates to the queried - ClinGen id. ``preferred_classification`` is the readable functional classification the UI defaults to - (primary-first cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). + ``assay_level`` is where this measurement was assayed (always shown, the clinically load-bearing + fact). ``preferred_classification`` is the readable classification the UI defaults to (primary-first + cascade, RUO excluded; ``None`` when none applies or the calibration is unreadable). """ variant_urn: str @@ -87,22 +89,27 @@ class AlleleMeasurement: superseded_by_score_set: Optional[str] +@dataclass(frozen=True) +class _ProteinApex: + """Protein consequence(s) co-membered with a CA anchor's nt alleles — the shared node a CA query + reaches protein measurements through. One PAID is well-formed; ``len(caids) > 1`` is a data-quality + signal (a duplicate or divergent consequence). ``unregistered`` counts apex rows without a PAID yet. + """ + + allele_ids: list[int] + caids: list[str] + unregistered: int + + def _preferred_classification( db: Session, variant: Variant, *, user_data: Optional[UserData] ) -> Optional[ScoreCalibrationFunctionalClassification]: """The variant's preferred *readable* functional classification, or ``None``. - A variant falls into one classification per calibration (non-overlapping ranges); we surface the one - the UI would default to, mirroring its calibration preference cascade: ``primary`` first, then - ``investigator_provided``. Within a tier we take the strongest evidence (the VA-Spec posture — the - strongest calibration is the evidence calibration), then ``id`` only for determinism. - - This is a clinical surface, so research-use-only calibrations are excluded outright (not merely deprioritized - as in the shared cascade) — a RUO call is never the classification shown here. - - Only calibrations the caller can read (calibration READ) are considered, so a private calibration is - skipped rather than blanking a readable call. Kept in sync with the UI's `activeCalibrationOptions` - default selection. + Mirrors the UI's calibration cascade — ``primary`` then ``investigator_provided``, strongest evidence + within a tier, then ``id`` for determinism. RUO calibrations are excluded outright (never shown here), + and calibrations the caller can't read are skipped rather than blanking a readable call. Kept in sync + with the UI's ``activeCalibrationOptions`` default. """ candidates = [ (classification, calibration) @@ -134,13 +141,8 @@ def preference(pair: tuple[ScoreCalibrationFunctionalClassification, ScoreCalibr def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date]) -> tuple: - """Sort order: - 1. Current measurements first, then superseded (the latter only if ``include_superseded``) - 2. Direct measurements first, then related (protein_consequence / nucleotide_encoding) - 3. Within each, the strongest evidence first (pathogenic wins ties) - 4. Within each, the newest-published first - 5. Within each, the URN for a stable tiebreak - """ + """Sort: current before superseded; direct before related; strongest evidence (pathogenic wins ties) + first; newest-published first; then URN for a stable tiebreak.""" magnitude, direction = classification_evidence_strength(measurement.preferred_classification) return ( 0 if measurement.is_current else 1, @@ -153,22 +155,45 @@ def _ordering_key(measurement: AlleleMeasurement, published_date: Optional[date] ) +def _resolve_protein_apex(db: Session, anchor_ids: list[int], *, as_of: Optional[datetime]) -> _ProteinApex: + """The protein consequence(s) co-membered with the nt ``anchor_ids`` via a shared live record. + + The single site the apex is resolved, so its cardinality (see :class:`_ProteinApex`) is measured + rather than being a silent side effect of the union. Reads ``clingen_allele_id`` to count PAIDs. + """ + anchor_link = aliased(MappingRecordAllele) + rows = db.execute( + select(Allele.id, Allele.clingen_allele_id) + .join(MappingRecordAllele, MappingRecordAllele.allele_id == Allele.id) + .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) + .where(anchor_link.allele_id.in_(anchor_ids)) + .where(anchor_link.live_at(as_of)) + .where(MappingRecordAllele.live_at(as_of)) + .where(Allele.level == SequenceLevel.protein.value) + .distinct() + ).all() + + return _ProteinApex( + allele_ids=[row.id for row in rows], + caids=sorted({row.clingen_allele_id for row in rows if row.clingen_allele_id is not None}), + unregistered=sum(1 for row in rows if row.clingen_allele_id is None), + ) + + def get_allele_measurements( db: Session, clingen_allele_id: str, *, user_data: Optional[UserData], include_superseded: bool = False, - include_nucleotide_siblings: bool = False, as_of: Optional[datetime] = None, ) -> list[AlleleMeasurement]: - """List the measurements in ``clingen_allele_id``'s cross-layer equivalence class. Returns - ``[]`` when the id resolves to no allele or no live record. + """The measurements in ``clingen_allele_id``'s equivalence class, or ``[]`` if it resolves to no live + record. - ``include_nucleotide_siblings`` (a ``CA`` entry only; a no-op for ``PA``) widens the class through the - queried change's protein consequence to also surface the *sibling* nt changes — other nucleotide - variants that encode the same amino-acid change and were themselves assayed at the nucleotide level - (``relationship=nucleotide_encoding``). + A CA query returns ``direct`` measurements, its ``protein_consequence`` (reached through the protein + apex, see the module docstring), and the ``nucleotide_encoding`` siblings. A PA query returns its + ``direct`` protein measurements and their ``nucleotide_encoding`` encodings. """ anchor = db.execute(select(Allele.id, Allele.level).where(Allele.clingen_allele_id == clingen_allele_id)).all() if not anchor: @@ -177,22 +202,22 @@ def get_allele_measurements( anchor_ids = [row.id for row in anchor] entry_is_protein = any(row.level == SequenceLevel.protein.value for row in anchor) - # Sibling nucleotide changes (search discovery, CA only): fold the queried change's protein consequence - # — the protein alleles co-membered with the anchor nt alleles — into the anchor, so the single record - # union below also reaches every record encoding that consequence. a PA query already anchors on the protein, so - # this is exactly what makes a CA+siblings query behave like one. (This query would be redundant for a PA entry). - if include_nucleotide_siblings and not entry_is_protein: - anchor_link = aliased(MappingRecordAllele) - anchor_ids += db.scalars( - select(MappingRecordAllele.allele_id) - .join(anchor_link, anchor_link.mapping_record_id == MappingRecordAllele.mapping_record_id) - .join(Allele, Allele.id == MappingRecordAllele.allele_id) - .where(anchor_link.allele_id.in_(anchor_ids)) - .where(anchor_link.live_at(as_of)) - .where(MappingRecordAllele.live_at(as_of)) - .where(Allele.level == SequenceLevel.protein.value) - .distinct() - ).all() + # A CA query always includes its protein consequence, reached through the shared protein node, so it is + # reachable even when a protein assay isn't reverse-translated yet (its record then links only the + # protein node). Transitive, so it's measured: cardinality >1 logged, provenance tallied below. No-op + # for a PA (already anchored on the protein). + apex: Optional[_ProteinApex] = None + if not entry_is_protein: + apex = _resolve_protein_apex(db, anchor_ids, as_of=as_of) + anchor_ids += apex.allele_ids + if len(apex.caids) > 1: + logger.warning( + msg=( + f"ClinGen allele {clingen_allele_id} resolved to {len(apex.caids)} distinct protein " + f"apexes ({', '.join(apex.caids)}); measurements may conflate consequences." + ), + extra=logging_context(), + ) record_ids = db.scalars( select(MappingRecordAllele.mapping_record_id) @@ -203,6 +228,22 @@ def get_allele_measurements( if not record_ids: return [] + # Records that carry a derived (RT) link of their own. A protein consequence whose record has none was + # reached only through the apex of an already resolved protein consequence. + records_with_derived_links: set[int] = ( + set( + db.scalars( + select(MappingRecordAllele.mapping_record_id) + .where(MappingRecordAllele.mapping_record_id.in_(record_ids)) + .where(MappingRecordAllele.is_authoritative.is_(False)) + .where(MappingRecordAllele.live_at(as_of)) + .distinct() + ).all() + ) + if apex is not None + else set() + ) + # Each record's authoritative (measured) allele fixes the assayed level and the direct/related call. authoritative = aliased(MappingRecordAllele) rows = ( @@ -227,12 +268,14 @@ def get_allele_measurements( ) measurements: list[tuple[AlleleMeasurement, Optional[date]]] = [] + protein_consequence_count = 0 + apex_only_unresolved_count = 0 for record, variant, measured_allele in rows: score_set = variant.score_set if not has_permission(user_data, score_set, Action.READ).permitted: continue - # Don't leak unpermitted resources. + # Don't leak an unreadable superseding score set. superseding = score_set.superseding_score_set if superseding is not None and not has_permission(user_data, superseding, Action.READ).permitted: superseding = None @@ -241,9 +284,7 @@ def get_allele_measurements( if not is_current and not include_superseded: continue - # Label by the measured level, not the entry level: direct when the measured allele *is* the query, - # else protein→protein_consequence / nucleotide→nucleotide_encoding. Without siblings the sibling-nt - # branch is unreachable (a record links the anchor only via itself or its protein consequence). + # Label by the measured (authoritative) allele's level relative to the query. if measured_allele.clingen_allele_id == clingen_allele_id: relationship = MeasurementRelationship.direct elif measured_allele.level == SequenceLevel.protein.value: @@ -251,6 +292,13 @@ def get_allele_measurements( else: relationship = MeasurementRelationship.nucleotide_encoding + # Apex-only provenance: a protein consequence whose record has no derived link was reached purely + # through the apex. + if apex is not None and relationship == MeasurementRelationship.protein_consequence: + protein_consequence_count += 1 + if record.id not in records_with_derived_links: + apex_only_unresolved_count += 1 + assay_level = SequenceLevel(record.assay_level) if record.assay_level else None measurement = AlleleMeasurement( variant_urn=variant.urn or "", @@ -267,5 +315,16 @@ def get_allele_measurements( ) measurements.append((measurement, score_set.published_date)) + # Publish the apex provenance to the request log so the ambiguous-apex and apex-only-unresolved rates are observable. + if apex is not None: + save_to_logging_context( + { + "apex_paid_count": len(apex.caids), + "apex_unregistered_count": apex.unregistered, + "measurements_protein_consequence": protein_consequence_count, + "measurements_apex_only_unresolved": apex_only_unresolved_count, + } + ) + measurements.sort(key=lambda pair: _ordering_key(pair[0], pair[1])) return [measurement for measurement, _ in measurements] diff --git a/src/mavedb/routers/clingen_alleles.py b/src/mavedb/routers/clingen_alleles.py index 64c6c1544..b3b626c02 100644 --- a/src/mavedb/routers/clingen_alleles.py +++ b/src/mavedb/routers/clingen_alleles.py @@ -50,16 +50,6 @@ def get_clingen_allele_measurements( "measurements are a deliberate power-user / citation path, never surfaced by discovery." ), ), - include_nucleotide_siblings: bool = Query( - default=False, - description=( - "For a nucleotide (CA) query only: also return sibling nucleotide changes — other DNA " - "variants encoding the same protein consequence that were themselves assayed at the " - "nucleotide level (relationship 'nucleotide_encoding'). Default false: the variant page " - "anchors strictly on the queried allele. Discovery surfaces (search) set it to surface all " - "evidence bearing on the consequence. No-op for a protein (PA) query." - ), - ), as_of: Optional[datetime] = Query( default=None, description=( @@ -82,7 +72,6 @@ def get_clingen_allele_measurements( "requested_resource": clingen_allele_id, "as_of": as_of, "include_superseded": include_superseded, - "include_nucleotide_siblings": include_nucleotide_siblings, } ) response.headers["X-As-Of"] = as_of.isoformat() if as_of is not None else "current" @@ -92,6 +81,5 @@ def get_clingen_allele_measurements( clingen_allele_id, user_data=user_data, include_superseded=include_superseded, - include_nucleotide_siblings=include_nucleotide_siblings, as_of=as_of, ) diff --git a/tests/lib/test_allele_measurements.py b/tests/lib/test_allele_measurements.py index a4621c33b..319caea27 100644 --- a/tests/lib/test_allele_measurements.py +++ b/tests/lib/test_allele_measurements.py @@ -1,26 +1,31 @@ # ruff: noqa: E402 """Integration tests for the ClinGen-allele measurements list (``lib/allele_measurements.py``). -These pin the model that the first pass got wrong: measurements aggregate the **cross-layer equivalence -class** (co-membership in the mapping/RT link graph), not one exact allele, and *which* allele anchors it -determines the direct/related split (a CA anchors on the nt alleles → nt measurements are direct, the -protein consequence is related; a PA anchors on the protein → protein measurements are direct, the nt -encodings related). Also: the narrowness (a sibling nt change is not pulled onto a CA page), the two -authorization gates (score-set READ hides the measurement; calibration READ withholds only the inline -classification) exercised through the real ``has_permission``, superseded opt-in, and the default ordering -(direct-first, strongest evidence, pathogenic-biased). +These pin the model: measurements aggregate the **cross-layer equivalence class** (co-membership in the +mapping/RT link graph), not one exact allele, and *which* allele anchors it determines the direct/related +split (a CA anchors on the nt alleles → nt measurements are direct, the protein consequence and the sibling +nt encodings are related; a PA anchors on the protein → protein measurements are direct, the nt encodings +related). Also: the protein apex reaches a consequence whose assay isn't reverse-translated (with its +cardinality/provenance instrumentation), the two authorization gates (score-set READ hides the measurement; +calibration READ withholds only the inline classification) exercised through the real ``has_permission``, +superseded opt-in, and the default ordering (direct-first, strongest evidence, pathogenic-biased). """ +import logging + import pytest pytest.importorskip("psycopg2") from datetime import date +from starlette_context import context, request_cycle_context + from mavedb.lib.allele_measurements import ( AlleleMeasurement, MeasurementRelationship, _ordering_key, + _resolve_protein_apex, get_allele_measurements, ) from mavedb.lib.types.authentication import UserData @@ -204,26 +209,33 @@ def test_pa_entry_swaps_direct_and_related(session, setup_lib_db_with_score_set) @pytest.mark.integration -def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set): - """A sibling nt change that encodes the *same* protein is not pulled onto a CA page (its record links - the protein but not this nt allele) — but both nt changes show on the protein's PA page as encodings.""" +def test_sibling_nt_shown_on_ca_page(session, setup_lib_db_with_score_set): + """Reverse translation links every record to its full synonymous nt set, so a sibling nt change's record + links the anchor allele directly and the sibling shows on the CA page as ``nucleotide_encoding`` (after + the direct measurement). Both nt changes also show on the shared protein's PA page as encodings.""" score_set = setup_lib_db_with_score_set nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + # Real RT linking: each record links its authoritative nt allele, the sibling nt allele (an RT + # candidate), and the shared protein consequence. a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) ra = _record(session, a, assay_level="cdna") _link(session, ra, nt1, is_authoritative=True) + _link(session, ra, nt2) _link(session, ra, prot) c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) rc = _record(session, c, assay_level="cdna") _link(session, rc, nt2, is_authoritative=True) + _link(session, rc, nt1) _link(session, rc, prot) ca_page = get_allele_measurements(session, "CA111", user_data=_user_data(session)) - assert {m.variant_urn for m in ca_page} == {a.urn} + assert {m.variant_urn: m.relationship for m in ca_page} == {a.urn: "direct", c.urn: "nucleotide_encoding"} + # Direct precedes the sibling encoding. + assert [m.variant_urn for m in ca_page] == [a.urn, c.urn] pa_page = get_allele_measurements(session, "PA9", user_data=_user_data(session)) assert {m.variant_urn for m in pa_page} == {a.urn, c.urn} @@ -231,36 +243,132 @@ def test_sibling_nt_not_pulled_onto_ca_page(session, setup_lib_db_with_score_set @pytest.mark.integration -def test_sibling_nt_pulled_onto_ca_page_with_flag(session, setup_lib_db_with_score_set): - """``include_nucleotide_siblings`` widens a CA page through the protein consequence: the sibling nt - change encoding the same protein is pulled in as a ``nucleotide_encoding`` while the direct measurement - is unchanged. A no-op for a PA query, which already returns every encoding.""" +def test_resolve_protein_apex_single(session, setup_lib_db_with_score_set): + """The resolver returns the one protein consequence co-membered with the anchor nt allele, its PAID, + and no unregistered rows.""" score_set = setup_lib_db_with_score_set - nt1 = _allele(session, "nt-1", level="cdna", clingen_allele_id="CA111") - nt2 = _allele(session, "nt-2", level="cdna", clingen_allele_id="CA222") + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) - a = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) - ra = _record(session, a, assay_level="cdna") - _link(session, ra, nt1, is_authoritative=True) - _link(session, ra, prot) + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == ["PA9"] + assert apex.unregistered == 0 - c = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}) - rc = _record(session, c, assay_level="cdna") - _link(session, rc, nt2, is_authoritative=True) - _link(session, rc, prot) - result = get_allele_measurements(session, "CA111", user_data=_user_data(session), include_nucleotide_siblings=True) - by_urn = {m.variant_urn: m for m in result} - assert set(by_urn) == {a.urn, c.urn} - assert by_urn[a.urn].relationship == "direct" - assert by_urn[c.urn].relationship == "nucleotide_encoding" - # Display order: the directly-measured change precedes its sibling nucleotide encoding. - assert [m.variant_urn for m in result] == [a.urn, c.urn] - - # No-op for a protein anchor — it already returns every nt encoding. - pa = get_allele_measurements(session, "PA9", user_data=_user_data(session), include_nucleotide_siblings=True) - assert {m.variant_urn for m in pa} == {a.urn, c.urn} +@pytest.mark.integration +def test_resolve_protein_apex_unregistered(session, setup_lib_db_with_score_set): + """An apex protein allele with no CAID yet counts toward ``unregistered`` and contributes no PAID.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id=None) + v = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r = _record(session, v, assay_level="cdna") + _link(session, r, nt, is_authoritative=True) + _link(session, r, prot) + + apex = _resolve_protein_apex(session, [nt.id], as_of=None) + assert apex.allele_ids == [prot.id] + assert apex.caids == [] + assert apex.unregistered == 1 + + +@pytest.mark.integration +def test_widening_ambiguous_apex_is_loud(session, setup_lib_db_with_score_set, caplog): + """An anchor whose nt change is co-membered with two *different* protein apexes (a divergent walk) is + surfaced loudly: a warning naming the conflicting PAIDs, and ``apex_paid_count == 2`` in the request + log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot1 = _allele(session, "prot-1", level="protein", clingen_allele_id="PA1") + prot2 = _allele(session, "prot-2", level="protein", clingen_allele_id="PA2") + + # nt is the measured allele of r1 (→ PA1) and a derived member of r2 (→ PA2): the same nt change + # resolving to two protein consequences. + v1 = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}) + r1 = _record(session, v1, assay_level="cdna") + _link(session, r1, nt, is_authoritative=True) + _link(session, r1, prot1) + + v2 = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + r2 = _record(session, v2, assay_level="protein") + _link(session, r2, prot2, is_authoritative=True) + _link(session, r2, nt) + + with request_cycle_context({}): + with caplog.at_level(logging.WARNING, logger="mavedb.lib.allele_measurements"): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["apex_paid_count"] == 2 + + assert any("PA1" in rec.message and "PA2" in rec.message for rec in caplog.records) + + +@pytest.mark.integration +def test_widening_provenance_apex_only_unresolved(session, setup_lib_db_with_score_set): + """A protein measurement reached only through the apex, whose own record carries no derived link, is + counted apex-only-unresolved; giving that record a derived link of its own drops the count.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": 1.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + # Both surface via the apex; the protein measurement is the apex-only-unresolved one. + assert {m.variant_urn for m in result} == {nt_measured.urn, protein_measured.urn} + assert context["measurements_protein_consequence"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 + + # Once the protein record gains a derived (non-authoritative) link of its own, it is resolved. + _link(session, protein_record, nt) + with request_cycle_context({}): + get_allele_measurements(session, "CA123", user_data=_user_data(session)) + assert context["measurements_apex_only_unresolved"] == 0 + + +@pytest.mark.integration +def test_ca_page_reaches_apex_only_protein_consequence(session, setup_lib_db_with_score_set): + """The variant page (no siblings) reaches a protein measurement of the anchor's consequence even when + that measurement's record has no nt link of its own (RT never ran on its score set) — through the + shared apex — and publishes the apex provenance to the request log.""" + score_set = setup_lib_db_with_score_set + nt = _allele(session, "nt-N", level="cdna", clingen_allele_id="CA123") + prot = _allele(session, "prot-P", level="protein", clingen_allele_id="PA9") + + # nt measurement: RT ran — its record links nt (authoritative) and the protein consequence. + nt_measured = _variant(session, score_set, 1, data={"score_data": {"score": -2.0}}, hgvs_nt="c.1A>T") + nt_record = _record(session, nt_measured, assay_level="cdna") + _link(session, nt_record, nt, is_authoritative=True) + _link(session, nt_record, prot) + + # protein measurement: RT has NOT run — its record links only its authoritative protein allele. + protein_measured = _variant(session, score_set, 2, data={"score_data": {"score": 1.0}}, hgvs_pro="p.Met1Leu") + protein_record = _record(session, protein_measured, assay_level="protein") + _link(session, protein_record, prot, is_authoritative=True) + + with request_cycle_context({}): + result = get_allele_measurements(session, "CA123", user_data=_user_data(session)) + by_urn = {m.variant_urn: m for m in result} + # Reached through the apex despite no nt link on the protein measurement's record. + assert set(by_urn) == {nt_measured.urn, protein_measured.urn} + assert by_urn[protein_measured.urn].relationship == "protein_consequence" + # The page path now widens, so it emits the apex instrumentation. + assert context["apex_paid_count"] == 1 + assert context["measurements_apex_only_unresolved"] == 1 @pytest.mark.integration diff --git a/tests/routers/test_clingen_allele.py b/tests/routers/test_clingen_allele.py index 8c01ec8ef..7ac06e709 100644 --- a/tests/routers/test_clingen_allele.py +++ b/tests/routers/test_clingen_allele.py @@ -108,10 +108,9 @@ def test_superseded_measurement_is_opt_in(client, session, data_provider, data_f assert body[0]["supersededByScoreSet"] == newer["urn"] -def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_files, setup_router_db): - """The sibling nt bucket is a discovery opt-in: a different DNA variant encoding the same protein - consequence is absent by default and pulled in as a ``nucleotide_encoding`` under - ``include_nucleotide_siblings``.""" +def test_nucleotide_siblings_shown(client, session, data_provider, data_files, setup_router_db): + """A different DNA variant encoding the same protein consequence surfaces on a CA page as a + ``nucleotide_encoding``, reached through the shared protein consequence.""" experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -133,20 +132,17 @@ def test_nucleotide_siblings_are_opt_in(client, session, data_provider, data_fil session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) session.commit() - default = client.get("/api/v1/clingen-alleles/CA111/measurements") - assert {m["variantUrn"] for m in default.json()} == {f"{score_set['urn']}#1"} - - widened = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) - assert widened.status_code == 200 - by_urn = {m["variantUrn"]: m for m in widened.json()} + response = client.get("/api/v1/clingen-alleles/CA111/measurements") + assert response.status_code == 200 + by_urn = {m["variantUrn"]: m for m in response.json()} assert set(by_urn) == {f"{score_set['urn']}#1", f"{score_set['urn']}#2"} assert by_urn[f"{score_set['urn']}#1"]["relationship"] == "direct" assert by_urn[f"{score_set['urn']}#2"]["relationship"] == "nucleotide_encoding" def test_direct_measurements_sort_before_related(client, session, data_provider, data_files, setup_router_db): - """The list is ordered, and direct measurements precede related ones in the body: a CA+siblings query - returns the directly-assayed change first, then the sibling nucleotide encoding.""" + """The list is ordered, and direct measurements precede related ones in the body: a CA query returns + the directly-assayed change first, then the sibling nucleotide encoding.""" experiment = create_experiment(client) score_set = create_seq_score_set_with_variants( client, session, data_provider, experiment["urn"], data_files / "scores.csv" @@ -167,7 +163,7 @@ def test_direct_measurements_sort_before_related(client, session, data_provider, session.add(MappingRecordAllele(mapping_record_id=record.id, allele_id=prot.id)) session.commit() - response = client.get("/api/v1/clingen-alleles/CA111/measurements", params={"include_nucleotide_siblings": True}) + response = client.get("/api/v1/clingen-alleles/CA111/measurements") assert response.status_code == 200 body = response.json() From a9c2f404f672261f394e97ced7063e5823f20189 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 24 Jul 2026 08:00:28 -0700 Subject: [PATCH 93/93] feat(cleanup): extend job timeouts Even with higher timeout settings, some VEP jobs were still hitting against the timeout window. Given the new checkpoint features, increasing the coarse timeout is lower risk and, in the aggregate, will help more jobs run to completion. --- src/mavedb/worker/jobs/system/cleanup.py | 6 +++--- src/mavedb/worker/settings/worker.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mavedb/worker/jobs/system/cleanup.py b/src/mavedb/worker/jobs/system/cleanup.py index 4a2dc975e..40e636be6 100644 --- a/src/mavedb/worker/jobs/system/cleanup.py +++ b/src/mavedb/worker/jobs/system/cleanup.py @@ -46,11 +46,11 @@ # how long it has legitimately been running. This is what lets a multi-hour VEP fan-out run to # completion without being killed, while still catching a job that has genuinely hung. # 2. RUNNING_TIMEOUT_MINUTES — a wall-clock backstop for jobs that never emit progress (or whose -# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (8h) so +# heartbeat column is somehow stuck). Sits ~30 min under ArqWorkerSettings.job_timeout (24h) so # the DB-driven sweeper, not ARQ's hard kill, is what terminates a job — keeping DB state and # actual work in sync. -PROGRESS_STALL_MINUTES = 20 # RUNNING jobs should checkpoint progress at least this often -RUNNING_TIMEOUT_MINUTES = 450 # Wall-clock backstop (7.5h), 30 min under the 8h ARQ ceiling +PROGRESS_STALL_MINUTES = 30 # RUNNING jobs should checkpoint progress at least this often +RUNNING_TIMEOUT_MINUTES = 1410 # Wall-clock backstop (23.5h), 30 min under the 24h ARQ ceiling PENDING_TIMEOUT_MINUTES = 5 # PENDING jobs which are actionable within pipelines should be enqueued within 5 minutes PIPELINE_STUCK_TIMEOUT_MINUTES = ( 5 # Pipelines in non-terminal states with no active jobs should resolve within 5 minutes diff --git a/src/mavedb/worker/settings/worker.py b/src/mavedb/worker/settings/worker.py index 3b0ab4e89..f205fae7e 100644 --- a/src/mavedb/worker/settings/worker.py +++ b/src/mavedb/worker/settings/worker.py @@ -25,13 +25,13 @@ # driver, enabling incremental migration of job functions without touching # the FastAPI layer. Once all jobs use async sessions, raise MAX_JOBS to 10+. MAX_JOBS = 2 -# ARQ's hard coroutine-kill ceiling. Kept deliberately high (8h) so ARQ is the *last* resort: +# ARQ's hard coroutine-kill ceiling. Kept deliberately high (24h) so ARQ is the *last* resort: # our own DB-driven sweeper (cleanup_stalled_jobs) should detect and recover stalls long before # this fires, keeping the JobRun state machine in sync with reality. VEP fan-out over large allele # sets can legitimately run for hours, so a low ceiling would kill healthy work. # cleanup's RUNNING_TIMEOUT_MINUTES backstop sits ~30 min under this to try and sweep these jobs before # ARQ's hard timeout fires. -JOB_TIMEOUT_SECONDS = 8 * 60 * 60 # 8 hours +JOB_TIMEOUT_SECONDS = 24 * 60 * 60 # 24 hours class ArqWorkerSettings: