From 3f29059e0d940aa7d8a762eee7abc3281d8e8f95 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 16:57:24 -0400 Subject: [PATCH 01/42] Apollo release package: fix jbrowseOrganismList, add design spec jbrowseOrganismList did not compile: line 11 passed an undeclared $organismAbbrev to the JBrowseUtil constructor under use strict, so /service/jbrowse/organismList returned a Perl error with HTTP 200. getDbh only needs projectName. Two further defects in that script are documented but not yet fixed (they need the same pass): $historySql joins to a nonexistent alias `nt`, and addHistoryToOrganism reads Oracle-cased hash keys that DBD::Pg returns lowercase -- so HISTORY is silently always absent. Spec covers rebuilding createApolloReleasePackage against the UniDB portal on cedar, seeded from live prod Apollo rather than a frozen build-68 organism list. Co-Authored-By: Claude Opus 5 --- Model/bin/jbrowseOrganismList | 2 +- ...026-08-21-apollo-release-package-design.md | 287 ++++++++++++++++++ 2 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/specs/2026-08-21-apollo-release-package-design.md diff --git a/Model/bin/jbrowseOrganismList b/Model/bin/jbrowseOrganismList index 37d39e764a..06c291f5f7 100644 --- a/Model/bin/jbrowseOrganismList +++ b/Model/bin/jbrowseOrganismList @@ -8,7 +8,7 @@ use Data::Dumper; my ($projectName) = @ARGV; -my $jbrowseUtil = ApiCommonModel::Model::JBrowseUtil->new({projectName => $projectName, organismAbbrev => $organismAbbrev}); +my $jbrowseUtil = ApiCommonModel::Model::JBrowseUtil->new({projectName => $projectName}); my $dbh = $jbrowseUtil->getDbh(); my $sql = "select distinct o.public_abbrev as organism_abbrev diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md new file mode 100644 index 0000000000..3b6e7ff967 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -0,0 +1,287 @@ +# Apollo release package generation from the UniDB portal + +**Date:** 2026-08-21 +**Repos changed:** `ApiCommonModel` +**Branch:** `feat/apollo-configs` +**Instance for QA:** eupathdb (`~/workspaces/eupathdb`), model `UniDB`, appDb `genomicsdb_071n` +**Replaces:** `~/apollo_config/bld-71-createApolloReleasePackage_ALL.pl` (Paul Wilkinson's +script) and its `Paul_Wilkinson_Handover_notes.docx`. + +## 1. Goal + +Each release, VEuPathDB must publish JBrowse configuration and sequence data for the +organisms hosted in Apollo, the genome curation platform. The existing script has not +produced a usable package since build 68, and its author has left the project. + +Rebuild it as a maintained tool in `ApiCommonModel` that: + +- runs on **cedar** against the **UniDB portal**, calling the `jbrowse*` scripts directly + rather than scraping eleven component websites over HTTP; +- treats **live prod Apollo** as the roster's starting point, so curators' decisions are + not silently reverted each build; +- reports what will be **added, pruned, and renamed** before anything is generated, + because today nothing computes prunes at all; +- never mutates Apollo. It writes command files for a human to run, as today. + +## 2. What is actually broken + +Every row measured on 2026-08-21, not inferred. + +| # | Fact | Evidence | +|---|---|---| +| 1 | `jbrowseOrganismList` does not compile | Line 11 passes `$organismAbbrev` to the `JBrowseUtil` constructor; the variable is never declared and the file is `use strict`. Fails identically over HTTP and on the command line. | +| 2 | Its history query is malformed | `$historySql` ends `and o.taxon_id = nt.taxon_id`; there is no `nt` in its FROM clause. `DBD::Pg` throws, the script exits **0**, and prints JSON with no `HISTORY` key. | +| 3 | The script reads Oracle-cased keys on Postgres | `addHistoryToOrganism` reads `$h->{PUBLIC_ABBREV}` / `$_->{ORGANISM_ABBREV}`; `DBD::Pg` returns lowercase. | +| 4 | Consequently Paul's script processes **zero** organisms | It reads `$organism->{IS_REFERENCE_STRAIN}` etc., all now `undef`, so every organism falls out at `next unless`. `/eupath/data/apolloConfigs/release-71/` on yew contains one empty `AmoebaDB/` dir, 0 data dirs, 0 twoBit, 0 update commands. | +| 5 | Releases 69 and 70 were never produced | `/eupath/data/apolloConfigs/` holds release-64…68, then 71 (empty). | +| 6 | `__DATA__` is a frozen build-68 snapshot | Its newest entries have `first_build = 68`; nothing from 69, 70, 71. 33 of the 48 organisms it "excludes" did not exist when it was last edited. | +| 7 | Its remaining exclusions are the `@databases` array, not the list | The other 15 are host genomes (`hsapREF`, `mmusC57BL6J`, `btauHereford`, `scerS288C`, …). Paul's array names 11 component DBs and omits HostDB and SchistoDB. On the portal that filter does not exist and must be stated explicitly. | +| 8 | The update-commands pass ignores the data filters | It walks the whole organism list, so release-68/prod emitted 461 curl + 28 groovy commands against 472 data dirs. | +| 9 | Nothing computes prunes | Confirmed by inspection; the handover notes list no prune step. | +| 10 | Two organisms have been renamed and are orphaned | `cneoJEC21` → `cdenJEC21`, `cglaCBS138` → `nglaCBS138`. Both old abbrevs are still in prod Apollo and absent from the portal. | +| 11 | One of them holds curation work | Apollo id 2452162, `cneoJEC21`, **14 annotations**, `publicMode=true`. Prod Apollo holds 14,600 annotations across 102 organisms. | +| 12 | Species taxon ID cannot detect a rename | `cneoJEC21` was species 5207; `cdenJEC21` is 40410. `cglaCBS138` (5478 → *Nakaseomyces*) likewise. | +| 13 | The renamed genome is byte-identical | b68 `cneoJEC21.fa.fai` vs b71 `CdeneoformansJEC21/genome.fasta.fai`: 14 sequences, identical names and lengths. | +| 14 | `applicationType=apollo` is unimplemented | `Store::makeUrlTemplate` is `die "TODO: make apollo work"`. `getConfigurationObject` dispatches `apollo` to the abstract `getApolloObject`; **39** classes implement `getJBrowseObject`, **1** implements `getApolloObject`. | + +### Set sizes as of build 71 + +| set | count | +|---|---| +| portal organisms (`jbrowseOrganismList UniDB`) | 831 | +| reference **and** annotated | 501 | +| `__DATA__` | 471 | +| release-68/prod data dirs | 472 (471 + the `twoBit` symlink) | +| live prod Apollo | 459 | + +`__DATA__` and release-68/prod match exactly, so the list did govern generation. It +diverges from Apollo in both directions: 13 organisms it contains are absent from Apollo +(overwhelmingly VectorBase — consistent with the notes' rule that genomes curated +elsewhere are excluded), and it lacks 48 that now qualify. + +## 3. Roster model + +``` +roster(N) = apollo_live + + adds (approved, from the add-candidate bucket) + − removes (approved) + ± renames (repointed in place, never add+prune) +``` + +**Live prod Apollo is the seed**, not `__DATA__`. It is the only record of what curators +actually decided: it reflects the 13 removals `__DATA__` never captured, and it carries the +annotations that make a mistake expensive. + +`reference AND annotated` is demoted from a gate to a *proposal* input. It never adds or +removes anything on its own. + +`__DATA__` is retired. Its only irreducible content — organisms in Apollo that fail the +criteria — is recoverable from Apollo itself, and is 4 organisms, not 17: +`cposSilveira2022`, `lbraMHOMBR75M2904_2019`, `tbruLister427_2018`, `tcruYC6`. + +### Seeding the overlay + +Fifteen organisms qualify as reference+annotated, predate build 68, and have never been in +Apollo. Paul excluded them for free via an `@databases` array that omitted HostDB and +SchistoDB (fact 7); on the portal that filter does not exist, so without seeding they appear +as add candidates at every build forever and the bucket becomes noise nobody reads. They are +not one group, and must not be seeded as one: + +| organisms | disposition | +|---|---| +| `hsapREF`, `mmusC57BL6J`, `rnorBNNHsdMcwi`, `btauHereford`, `clupfamiliarisSID07034`, `ggalbGalGal1`, `cpor2N`, `mfasREF`, `mmulAG07107`, `mmyomMyoMyo1`, `dmeliso-1` | host genomes — seed as `remove`; Apollo curates pathogens | +| `scerS288C`, `spom972h` | model fungi, in FungiDB, never in Apollo — seed as `remove`, but flag to the curation team as a decision nobody has consciously made | +| `hcapNAm1` (*Histoplasma mississippiense* NAm1) | **not** a host genome. A FungiDB pathogen that qualifies and is absent from Apollo. Leave it in the add-candidate bucket for the curation team. | +| `cdenJEC21` | rename target; resolved by rename detection before the add bucket is computed. No overlay entry. | + +Recording these as `remove` with a stated reason is the point of the overlay: a decision made +once, with provenance, rather than an array that silently encoded it. The distinction above is +exactly what a project-name filter would have flattened — `hcapNAm1` would have been suppressed +along with the vertebrates, and nobody would ever have seen it. + +## 4. Inputs + +All read-only. + +| input | source | provides | +|---|---|---| +| portal organisms | `jbrowseOrganismList UniDB` on cedar | abbrev, name, `name_for_filenames`, `strain_abbrev`, ref/annotated flags, species taxon, `HISTORY` (after the §8 fixes) | +| live Apollo | `POST /organism/findAllOrganisms` from a Penn host | `id`, `commonName`, `directory`, `blatdb`, `annotationCount`, `publicMode` | +| roster overlay | `$GUS_HOME/data/ApiCommonModel/Model/apollo/roster-overlay.txt` | `add`/`remove` lines with approver and reason | +| genome files | `webServices/UniDB/build-NN//genomeAndProteome/fasta/` | `genome.fasta` + `genome.fasta.fai` | + +Verified: all 831 portal organisms have a directory and a `genome.fasta` under +`UniDB/build-71`. `samtools faidx` is unnecessary — the `.fai` ships alongside and contains +no self-reference, so renaming to `.fa` / `.fa.fai` is safe. + +The Apollo API is IP-restricted to Penn hosts. Credentials come from the environment +(`APOLLO_API_URL`, `APOLLO_API_USER`, `APOLLO_API_PASS`), never from a literal in the +source — Paul's script hardcodes a live password. + +## 5. Reconciliation + +Matching is on `organism_abbrev`, obtained from Apollo by parsing the trailing path +component of `directory` (`/data/apollo_data/tgonME49` → `tgonME49`). All 459 parse cleanly +with no duplicates. `directory` is machine-written by our own update commands; `commonName` +is editable in the Apollo GUI, so it is cross-checked and reported on mismatch, never used +for matching. + +Five buckets, with build-71 counts: + +| bucket | rule | count | action | +|---|---|---|---| +| update | in Apollo ∧ on portal | 457 | regenerate data, emit update command | +| add candidate | qualifies ∧ not in Apollo ∧ not `remove`d | 48 (33 new since b68) | **requires approval** | +| prune candidate | in Apollo ∧ not on portal ∧ not a rename | 2 → 0 after rename detection | **requires approval** | +| rename | see below | 2 | repoint in place | +| exception | in Apollo ∧ fails criteria | 4 | none; reported so it is not "fixed" | + +The exception bucket exists to prevent work, and to surface an organism *becoming* +non-qualifying rather than letting it persist unnoticed. + +### Rename detection + +For each Apollo organism whose abbrev is absent from the portal, compare the `.fa.fai` +shipped in the previous release against the `.fai` of each portal organism sharing its +`strain_abbrev`. An identical set of sequence names and lengths means the same assembly +under a new name. + +Species taxon ID is deliberately **not** used (fact 12). Strain abbrev narrows the +candidates; sequence identity decides. + +A rename emits **one** update against the existing Apollo `id`, setting `directory`, +`blatdb`, and `commonName`. Emitting an add for the new abbrev in the same run is a hard +error, not a convention: `add cdenJEC21` + `prune cneoJEC21` creates an empty organism and +destroys 14 annotations — same inputs, same API, opposite outcome. + +Renames whose sequence sets do **not** match fall through to prune-candidate and are +reported with their `annotationCount`, for a human. + +## 6. Generation + +Per organism in the approved roster: + +| output | produced by | +|---|---| +| `trackList.json` | assembled by the tool; `jbrowseTracks UniDB 0 geneAnnotationTracks` supplies the include list, rewritten to local filenames, with `tracks` replaced by the reference-sequence track | +| `tracks.conf` | `$GUS_HOME/lib/jbrowse/auto_generated//tracks.conf`, refseq stanza removed | +| `functions.conf` | `$GUS_HOME/lib/jbrowse/functions.conf` | +| `rnaseq.json`, `chipseq.json` | `jbrowseRnaAndChipSeqTracks UniDB {RNASeq,ChIPSeq} jbrowse` | +| `rnaseqJunctions.json` | `jbrowseRNASeqJunctionTracks UniDB 1 jbrowse` | +| `organismSpecific.json` | `jbrowseOrganismSpecificTracks UniDB 1 jbrowse` | +| `dnaseq.json` | `jbrowseDNASeqTracks UniDB jbrowse` | +| `seq/.fa`, `seq/.fa.fai` | copied and renamed from webServices | +| `seq/refSeqs.json` | `jbrowseRefSeqs $GUS_HOME UniDB ` | +| `twoBit/.2bit` | `faToTwoBit` — the only new computation | + +### URL absolutization + +Apollo needs absolute URLs; the scripts emit site-relative `/a/…`. We generate with +`applicationType=jbrowse` and run a **single text pass per generated file**, rewriting +`/a/` to `https://veupathdb.org/a/`, followed by an assertion that no bare `/a/` survives. + +`applicationType=apollo` is **not** used, and stays unimplemented (fact 14). It would need +`getApolloObject` in 39 classes, and even then would not cover the ~25 `/a/` literals that +live inside free text rather than in typed fields — HTML blobs (``), +`onClick` and `menuTemplate` URLs, `baseUrl` values, and JavaScript function bodies +(`"function(track,f) { … '/a/app/record/gene/' + f.get('name') }"`). A typed approach +cannot reach inside a JS string; a text pass covers all of it uniformly. + +This is deliberate, acknowledged debt. Paul did the same rewrite inline +(`s/\/a\//$website\/a\//g`) with no verification, so a missed URL became a track that +silently 404s inside Apollo. **The post-condition check is the part that makes this +approach safe, and is not optional.** + +The base is the portal, `https://veupathdb.org`, not a per-component site — one host now +serves every organism. + +No HTTP fetches, no `samtools`, and no dependency on the release being deployed to a public +site, so the package can be built as soon as webServices and the model exist — days earlier +than the current process permits. + +**Failures are per-organism.** Paul's script `die`s on the first missing input, discarding +hours of completed work. This tool records the failure, continues, prints a summary, and +exits non-zero. + +## 7. Output + +`~/apolloConfigs/release-NN//` on cedar (`--out-dir` overridable). `/eupath` is not +mounted on cedar; the rsync to yew is performed manually after inspection, and the tool does +not do it. + +Layout is unchanged from release-68, so systems' documented procedure still applies: + +``` +release-NN// + data// (symlinked into data/ as today) + twoBit/.2bit + updateCommands/{Apollo_curl,Apollo_groovy} + report.txt (new: the five buckets, as generated) +``` + +## 8. Packaging + +`ApiCommonModel/Model/bin/createApolloReleasePackage`, logic in +`Model/lib/perl/ApolloRelease/*.pm`, roster overlay in `Model/data/apollo/`. Installed to +`gus_home` by `bld ApiCommonModel/Model` (verified: `Model/data/` installs to +`$GUS_HOME/data/ApiCommonModel/Model/`). + +It lives here, not in `agentic-veupath-dev`, because it calls the `jbrowse*` scripts and must +version with them — an `applicationType` change has to move in one commit. The harness +invokes it as a `build.commands` entry. + +Two phases, because they cost differently: + +- `--report` — minutes. Portal + Apollo + overlay; prints the buckets. Safe at any time, + including before the release completes. This is what goes to the curation team. +- `--generate` — hours. Builds data and command files for the approved roster. + +Prerequisite fixes to `jbrowseOrganismList`, all three (nothing works without `HISTORY`): +the undeclared `$organismAbbrev` (done), the `nt.taxon_id` join corrected to +`dd.taxon_id = o.taxon_id` (verified: returns first/last build for all 831 organisms), and +the Oracle-cased keys in `addHistoryToOrganism`. + +## 9. Safety invariants + +1. The tool never calls a mutating Apollo endpoint. +2. Prune emits `publicMode=false`, not a delete — reversible, and Apollo's API has no delete + in use anyway. +3. Add and prune require an overlay entry; neither is inferred from the criteria alone. +4. A rename and an add for the same genome cannot both be emitted. +5. Every generated file passes the no-bare-`/a/` check before the run is declared successful. +6. Credentials come from the environment. The repo contains no password. + +## 10. Verification + +Exact reproduction of release-68 is impossible: `webServices/UniDB/` retains only `build-70` +and `build-71`, so the b68 inputs are gone. Instead: + +- **Structural diff** against release-68/prod for the ~450 organisms in both: identical file + set per organism dir, identical JSON keys, identical track counts and track ids. This + catches dropped includes and missing track categories. +- **Value diff** for organisms whose genome and datasets are unchanged between 68 and 71; + output should be near-identical, and any difference is a finding. URLs will differ by the + absolutization base and that difference must be exactly the base. +- **Unit tests** for reconciliation over fixture JSON: the two known renames, an annotated + prune candidate, an overlay add, and the mutual-exclusion rule of §5. +- **Sandbox before prod**, per the post-release checklist: gene track drags into the + annotation track, BLAT works against the new `.2bit`. Specifically exercise the + `cneoJEC21` → `cdenJEC21` repoint on the sandbox and confirm all 14 annotations survive + before it is ever run against prod. + +## 11. Deferred + +- **A real `apollo` application type.** When JBrowse2/Apollo3 arrives, `applicationType` + becomes the correct seam and §6's text pass retires. Not worth 39 classes now for output + that differs from `jbrowse` only in URL base. +- **JBrowse2 output.** The scripts accept `applicationType=jbrowse2`; nothing here uses it. +- **`datasetAndPresenterProps.conf` migration.** Paul's notes describe removing the DB handle + from the track scripts in favour of a per-organism props file. Partially landed upstream, + and independent of this work, which consumes those scripts' output either way. +- **Apollo `genus`/`species` fields.** Neither the curl nor the arrow command touches them, + so after a rename `commonName` will say *deneoformans* while those fields may not. + Determine whether the Apollo GUI keys off them; if so, this becomes part of the rename + command. +- **`--environment qa`.** Retained from the old script and still generated, but the qa and + prod rosters have never differed by anything but staleness. Revisit whether qa is worth + producing at all. From e6d0a68202bb4424f300f8c3b3779a1c43d29717 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 17:03:38 -0400 Subject: [PATCH 02/42] Apollo spec: record faToTwoBit resolution on cedar yew's faToTwoBit is a 2016 build against libssl.so.10 / libpng15, none of which exist on cedar (Rocky 9), so it cannot be copied. Installed UCSC's current linux.x86_64 build to ~/bin. Verified the two produce byte-identical output (md5 a8b9cb95a7794c1ed7dcc3568edeaff9) over the cneoJEC21/cdenJEC21 genome, which also confirms the rename at the byte level. Measured 0.17-0.74s per genome, so the spec now says always regenerate rather than copying .2bit forward between releases. Co-Authored-By: Claude Opus 5 --- ...026-08-21-apollo-release-package-design.md | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 3b6e7ff967..6a59ba0d83 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -111,7 +111,7 @@ All read-only. | genome files | `webServices/UniDB/build-NN//genomeAndProteome/fasta/` | `genome.fasta` + `genome.fasta.fai` | Verified: all 831 portal organisms have a directory and a `genome.fasta` under -`UniDB/build-71`. `samtools faidx` is unnecessary — the `.fai` ships alongside and contains +`UniDB/build-71` (0 missing dirs, 0 missing fastas). `samtools faidx` is unnecessary — the `.fai` ships alongside and contains no self-reference, so renaming to `.fa` / `.fa.fai` is safe. The Apollo API is IP-restricted to Penn hosts. Credentials come from the environment @@ -174,6 +174,30 @@ Per organism in the approved roster: | `seq/refSeqs.json` | `jbrowseRefSeqs $GUS_HOME UniDB ` | | `twoBit/.2bit` | `faToTwoBit` — the only new computation | +### faToTwoBit on cedar + +`faToTwoBit` existed only on yew, at `/eupath/workflow-software/bin/faToTwoBit` — a 2016 +build linked against `libssl.so.10`, `libcrypto.so.10`, and `libpng15.so.15`, none of which +exist on cedar (Rocky 9, glibc 2.34). Copying it across does not work. + +Resolved by installing UCSC's current `linux.x86_64` build to `~/bin/faToTwoBit` on cedar +(2026-08-21), which is already on PATH. Verified byte-for-byte against the old one: + +``` +release-68 twoBit/cneoJEC21.2bit (yew, 2016 binary) 4764116 a8b9cb95a7794c1ed7dcc3568edeaff9 +build-71 cdenJEC21.2bit (cedar, 2026 binary) 4764116 a8b9cb95a7794c1ed7dcc3568edeaff9 +``` + +Identical output from two binaries a decade apart, over the renamed organism's genome. This +also independently confirms fact 13 at the byte level. + +**Always regenerate; never copy forward.** Measured 0.17–0.74 s per genome, so the full +roster costs a couple of minutes. An incremental scheme keyed on genome version was +considered and rejected: it adds state to save nothing. + +The tool must check for `faToTwoBit` on PATH at startup and fail immediately with the +install instruction, rather than 400 organisms into a run. + ### URL absolutization Apollo needs absolute URLs; the scripts emit site-relative `/a/…`. We generate with From e1ee52ec87769b0363adb3de26e7e373453213b0 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 17:07:08 -0400 Subject: [PATCH 03/42] Apollo spec: replace release-68 diff with live-site equivalence Release-68 is not a valid baseline. Between b68 and b71 the track layer moved REST -> flat files, the database moved Oracle -> Postgres, and many track definitions changed; output matching release-68 would indicate a bug, not correctness. Verification is now equivalence with what the portal serves today for the same organism, where every difference must fall into one of four declared classes (absolutization, refseq track swap, include rewriting, deliberate removals). Adds a track-count floor and URL liveness sampling, since empty-but-valid JSON is the characteristic failure of the flat-file migration. Release-68 is retained only as the file-name inventory and for the faToTwoBit reproducibility check. Co-Authored-By: Claude Opus 5 --- ...026-08-21-apollo-release-package-design.md | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 6a59ba0d83..596b68bd14 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -277,15 +277,32 @@ the Oracle-cased keys in `addHistoryToOrganism`. ## 10. Verification -Exact reproduction of release-68 is impossible: `webServices/UniDB/` retains only `build-70` -and `build-71`, so the b68 inputs are gone. Instead: - -- **Structural diff** against release-68/prod for the ~450 organisms in both: identical file - set per organism dir, identical JSON keys, identical track counts and track ids. This - catches dropped includes and missing track categories. -- **Value diff** for organisms whose genome and datasets are unchanged between 68 and 71; - output should be near-identical, and any difference is a finding. URLs will differ by the - absolutization base and that difference must be exactly the base. +**Release-68 is not a baseline.** Between b68 and b71 the track layer moved from REST to +flat files, the database moved Oracle → Postgres, and a large number of track definitions +changed. Output that matched release-68 would be evidence of a *bug*, not of correctness. +Release-68 is useful for exactly two things: the file-name inventory a working organism dir +must contain (§6), and the `faToTwoBit` reproducibility check already done. + +The real baseline is **what the portal serves today**. Apollo's config and the site's JBrowse +config are generated from the same scripts against the same model; the Apollo variant should +differ only by known, enumerable transformations. So: + +- **Equivalence with the live site, per organism.** For a probe organism, generate the Apollo + package and diff each file against what the portal's own JBrowse serves for that organism. + Every difference must fall into one of four declared classes: + 1. URL absolutization (`/a/` → `https://veupathdb.org/a/`), + 2. the reference-sequence track replaced by the local `IndexedFasta` track, + 3. `include` URLs rewritten to local filenames, + 4. removals: the `[tracks.refseq]` stanza and `user-datasets-jbrowse` includes. + + A difference outside those four classes is a defect. This is the test that would have + caught Paul's silent skips, and it is meaningful *because* the tracks changed. +- **Track-count floor per organism.** Non-zero track count, and every `include` named in + `trackList.json` resolves to a file that exists and parses. Empty-but-valid JSON is the + characteristic failure of the flat-file migration and is invisible to a schema check. +- **URL liveness on a sample.** For a handful of organisms, HEAD every absolutized store URL + and require 200. The absolutization assertion proves the string changed; only a request + proves it points at something. - **Unit tests** for reconciliation over fixture JSON: the two known renames, an annotated prune candidate, an overlay add, and the mutual-exclusion rule of §5. - **Sandbox before prod**, per the post-release checklist: gene track drags into the @@ -293,6 +310,9 @@ and `build-71`, so the b68 inputs are gone. Instead: `cneoJEC21` → `cdenJEC21` repoint on the sandbox and confirm all 14 annotations survive before it is ever run against prod. +Probe organisms: `tgonME49` (rich track set, 10 annotations in Apollo), `pfal3D7` (different +component DB, partitioned queries), and `cdenJEC21` (the rename). + ## 11. Deferred - **A real `apollo` application type.** When JBrowse2/Apollo3 arrives, `applicationType` From 3ac106a4fb3baa5445e6dada8630e040865d2ed4 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 17:15:37 -0400 Subject: [PATCH 04/42] Apollo: add implementation plan 14 tasks, TDD throughout, against the 2026-08-21 design spec. Reconciliation is pure and fixture-tested; only Portal, Apollo, Generate and Commands touch the world. Co-Authored-By: Claude Opus 5 --- .../2026-08-21-apollo-release-package.md | 2451 +++++++++++++++++ 1 file changed, 2451 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-21-apollo-release-package.md diff --git a/docs/superpowers/plans/2026-08-21-apollo-release-package.md b/docs/superpowers/plans/2026-08-21-apollo-release-package.md new file mode 100644 index 0000000000..3bbeb685c3 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-apollo-release-package.md @@ -0,0 +1,2451 @@ +# Apollo Release Package Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace Paul Wilkinson's broken `createApolloReleasePackage_ALL.pl` with a maintained tool in `ApiCommonModel` that generates the Apollo release package from the UniDB portal on cedar, seeded from live prod Apollo. + +**Architecture:** A thin CLI (`Model/bin/createApolloReleasePackage`) over five focused Perl modules under `Model/lib/perl/ApolloRelease/`. Reconciliation (pure set logic over hashrefs) is separated from generation (filesystem + subprocess) so the former is unit-testable against fixtures with no database, no network, and no `GUS_HOME`. The tool computes five buckets, reports them, and — only for an approved roster — generates per-organism data dirs plus Apollo command files. It never mutates Apollo. + +**Tech Stack:** Perl 5 (`Test::More`, `prove`), `JSON`, `LWP::UserAgent`, the existing `ApiCommonModel::Model::JBrowseUtil` and `Model/bin/jbrowse*` scripts, UCSC `faToTwoBit`. + +**Spec:** `docs/superpowers/specs/2026-08-21-apollo-release-package-design.md`. Read it first; this plan assumes its facts and does not re-derive them. + +--- + +## Environment and conventions + +Everything runs on **cedar**, against the **eupathdb** instance, model **UniDB**. + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && "' +``` + +Local edits are made in `~/workspaces/eupathdb/ApiCommonModel` and reach cedar via mutagen. +**Do not create a git worktree** — mutagen syncs only the instance directory, so a worktree +elsewhere is invisible to cedar and every build would test stale code. + +Install after each change to `Model/`: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model"' +``` + +`bld` takes ~13 s. Do **not** run `wb model` — it OOMs on the UniDB portal. + +Module naming: `Model/lib/perl/ApolloRelease/Foo.pm` installs to +`$GUS_HOME/lib/perl/ApiCommonModel/Model/ApolloRelease/Foo.pm` and declares +`package ApiCommonModel::Model::ApolloRelease::Foo;`. + +Tests live in `Model/t/`, use `Test::More`, and run against the **installed** copy: + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/.t"' +``` + +So the loop for every task is: edit → `bld` → `prove`. + +There are no existing Perl tests in this repo; `Model/t/` is new. Fixtures live in +`Model/t/fixtures/` and are committed — they are small JSON files, and they are what makes +reconciliation testable without a database. + +**Branch:** `feat/apollo-configs`, already created, already carrying the +`jbrowseOrganismList` compile fix and the spec. + +--- + +## File structure + +| File | Responsibility | +|---|---| +| `Model/bin/jbrowseOrganismList` | *(modify)* fix the history join and the Oracle-cased keys | +| `Model/lib/perl/ApolloRelease/Portal.pm` | Run `jbrowseOrganismList`, normalise to organism hashrefs, expose `latest_annotation_version` | +| `Model/lib/perl/ApolloRelease/Apollo.pm` | Fetch live Apollo organisms, parse `abbrev` from `directory`, flag `commonName` disagreement | +| `Model/lib/perl/ApolloRelease/Overlay.pm` | Parse and validate `roster-overlay.txt` | +| `Model/lib/perl/ApolloRelease/Reconcile.pm` | Pure set logic: five buckets, rename detection, invariant enforcement | +| `Model/lib/perl/ApolloRelease/Report.pm` | Render the buckets as `report.txt` | +| `Model/lib/perl/ApolloRelease/Absolutize.pm` | `/a/` → `https://veupathdb.org/a/` plus the no-bare-`/a/` assertion | +| `Model/lib/perl/ApolloRelease/Generate.pm` | Per-organism file generation; per-organism failure isolation | +| `Model/lib/perl/ApolloRelease/Commands.pm` | Emit `Apollo_curl` / `Apollo_groovy` | +| `Model/bin/createApolloReleasePackage` | CLI: option parsing, preflight, wiring, exit status | +| `Model/data/apollo/roster-overlay.txt` | The seeded overlay | +| `Model/t/*.t` | Tests | + +Reconcile, Overlay, Absolutize, and Report have no I/O beyond their arguments. Portal, +Apollo, Generate, and Commands are the only modules that touch the world. + +--- + +## Task 1: Finish fixing `jbrowseOrganismList` + +The compile error is already fixed. Two defects remain, and `HISTORY` — which supplies the +annotation version in every Apollo organism name — is silently absent until both are done. + +**Files:** +- Modify: `Model/bin/jbrowseOrganismList:30-36` (the `$historySql`), `:60-70` (`addHistoryToOrganism`) + +- [ ] **Step 1: Confirm the current failure** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && \ + perl \$GUS_HOME/bin/jbrowseOrganismList UniDB > /tmp/ol.json 2>/tmp/ol.err; \ + echo rc=\$?; head -3 /tmp/ol.err; \ + python3 -c \"import json;d=json.load(open('/tmp/ol.json'))['organisms'];print('organisms',len(d));print('with HISTORY',sum(1 for o in d if 'HISTORY' in o or 'history' in o))\""' +``` + +Expected: `rc=0`, stderr containing `missing FROM-clause entry for table "nt"`, +`organisms 831`, `with HISTORY 0`. The zero is the bug: the script succeeds while producing +incomplete data. + +- [ ] **Step 2: Fix the history query's join** + +In `Model/bin/jbrowseOrganismList`, replace the `$historySql` assignment with: + +```perl +my $historySql = "select h.build_number, o.public_abbrev, h.genome_source, h.genome_version, h.annotation_source, h.annotation_version + from apidbtuning.datasethistory h, apidbtuning.datasetdatasource dd, apidb.organism o + where h.dataset_presenter_id = dd.dataset_presenter_id + and dd.name like '%primary_genome_RSRC' + and h.annotation_version is not null + and dd.taxon_id = o.taxon_id"; +``` + +The only change is the last line: `o.taxon_id = nt.taxon_id` → `dd.taxon_id = o.taxon_id`. +`nt` was never in the FROM clause; `datasetdatasource.taxon_id` is the real link to +`apidb.organism`. + +- [ ] **Step 3: Fix the Oracle-cased hash keys** + +`DBD::Pg` returns lowercase column names; `DBD::Oracle` returned uppercase. Replace +`addHistoryToOrganism` with: + +```perl +sub addHistoryToOrganism { + my ($h, $orgs) = @_; + my $publicAbbrev = $h->{public_abbrev}; + foreach(@$orgs) { + + if($_->{organism_abbrev} eq $publicAbbrev) { + push @{$_->{history}}, $h; + return; + } + } + die "Could not match organism abbrev $publicAbbrev"; +} +``` + +Note the pushed key is now `history`, matching the lowercase convention of every other key +the script emits. Consumers in this plan read `history`. + +- [ ] **Step 4: Install and verify** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && \ + perl \$GUS_HOME/bin/jbrowseOrganismList UniDB > /tmp/ol.json 2>/tmp/ol.err; \ + echo rc=\$?; echo err_bytes=\$(stat -c%s /tmp/ol.err); \ + python3 -c \"import json;d=json.load(open('/tmp/ol.json'))['organisms'];print('organisms',len(d));print('with history',sum(1 for o in d if o.get('history')));import collections;o=[x for x in d if x['organism_abbrev']=='tgonME49'][0];print('tgonME49 history rows',len(o['history']));print(o['history'][0])\""' +``` + +Expected: `rc=0`, **`err_bytes=0`**, `organisms 831`, `with history` well above 0, and a +sample history row containing `build_number` and `annotation_version`. + +`err_bytes=0` is the real assertion. This script is served through +`responseFromCommand`, which merges stderr into the JSON response body — so any warning +here becomes a corrupt API response, not just noise. + +- [ ] **Step 5: Save a fixture for later tasks** + +```bash +ssh cedar 'python3 -c " +import json +d=json.load(open(\"/tmp/ol.json\"))[\"organisms\"] +keep={\"tgonME49\",\"pfal3D7\",\"cdenJEC21\",\"nglaCBS138\",\"hcapNAm1\",\"hsapREF\",\"tbruLister427_2018\",\"tbruTREU927\"} +out=[o for o in d if o[\"organism_abbrev\"] in keep] +json.dump({\"organisms\":out}, open(\"/tmp/portal_fixture.json\",\"w\"), indent=1) +print(len(out)) +"' +scp cedar:/tmp/portal_fixture.json ~/workspaces/eupathdb/ApiCommonModel/Model/t/fixtures/portal.json +``` + +Expected: `8`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/bin/jbrowseOrganismList Model/t/fixtures/portal.json +git commit -m "fix(jbrowse): repair organismList history query and Postgres key casing + +The history query joined to an alias 'nt' absent from its FROM clause, +so it threw while the script still exited 0 and printed JSON with no +history. addHistoryToOrganism then read Oracle-cased keys that DBD::Pg +returns lowercase. Together these meant HISTORY was always missing, and +with it the annotation version every Apollo organism name depends on." +``` + +--- + +## Task 2: `ApolloRelease::Portal` + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Portal.pm` +- Create: `Model/t/portal.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/portal.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 9; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Portal; + +my $P = 'ApiCommonModel::Model::ApolloRelease::Portal'; + +my $orgs = $P->loadFromFile("Model/t/fixtures/portal.json"); + +is(ref($orgs), 'HASH', 'returns a hash keyed by abbrev'); +ok(exists $orgs->{tgonME49}, 'tgonME49 present'); + +my $t = $orgs->{tgonME49}; +is($t->{abbrev}, 'tgonME49', 'abbrev normalised'); +is($t->{name_for_filenames}, 'TgondiiME49', 'name_for_filenames carried'); +is($t->{is_reference}, 1, 'is_reference is a boolean, not a string'); +is($t->{is_annotated}, 1, 'is_annotated is a boolean, not a string'); + +my $b = $orgs->{tbruLister427_2018}; +is($b->{is_reference}, 0, 'non-reference strain is 0'); + +ok(defined $t->{latest_annotation_version}, 'latest annotation version derived'); +is($P->qualifies($b), 0, 'non-reference organism does not qualify'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/portal.t"' +``` + +Expected: FAIL — `Can't locate ApiCommonModel/Model/ApolloRelease/Portal.pm`. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Portal.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Portal; + +use strict; +use warnings; + +use JSON; + +# Reads the organism list produced by Model/bin/jbrowseOrganismList and +# normalises it. Keys arrive lowercase from DBD::Pg; booleans arrive as the +# strings "1"/"0". Everything downstream sees plain 0/1 and a single +# `abbrev` key. + +sub loadFromCommand { + my ($class, $projectName) = @_; + + my $gusHome = $ENV{GUS_HOME} or die "GUS_HOME is not set\n"; + my $cmd = "$gusHome/bin/jbrowseOrganismList $projectName"; + + my $json = `$cmd 2>/tmp/apolloRelease.organismList.err`; + die "jbrowseOrganismList failed (exit " . ($? >> 8) . "); see /tmp/apolloRelease.organismList.err\n" + if $?; + + my $errSize = -s "/tmp/apolloRelease.organismList.err" || 0; + die "jbrowseOrganismList wrote $errSize bytes to stderr; refusing to trust its output.\n" + . "See /tmp/apolloRelease.organismList.err\n" + if $errSize; + + return $class->_normalise(decode_json($json)); +} + +sub loadFromFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read $path: $!"; + local $/; + my $json = <$fh>; + close $fh; + + return $class->_normalise(decode_json($json)); +} + +sub _normalise { + my ($class, $decoded) = @_; + + my %byAbbrev; + + foreach my $raw (@{$decoded->{organisms}}) { + my $abbrev = $raw->{organism_abbrev}; + + $byAbbrev{$abbrev} = { + abbrev => $abbrev, + name => $raw->{name}, + name_for_filenames => $raw->{name_for_filenames}, + strain_abbrev => $raw->{strain_abbrev}, + species_taxon => $raw->{species_ncbi_tax_id}, + is_reference => $raw->{is_reference_strain} ? 1 : 0, + is_annotated => $raw->{is_annotated_genome} ? 1 : 0, + history => $raw->{history} || [], + latest_annotation_version => $class->_latestAnnotationVersion($raw->{history}), + }; + } + + return \%byAbbrev; +} + +# The annotation version belonging to the highest build number. Apollo names +# an organism " []", so this string is part of +# the organism's identity there. +sub _latestAnnotationVersion { + my ($class, $history) = @_; + + return undef unless $history && @$history; + + my $best; + foreach my $h (@$history) { + next unless defined $h->{annotation_version}; + $best = $h if !$best || $h->{build_number} > $best->{build_number}; + } + + return $best ? $best->{annotation_version} : undef; +} + +sub qualifies { + my ($class, $organism) = @_; + return ($organism->{is_reference} && $organism->{is_annotated}) ? 1 : 0; +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/portal.t"' +``` + +Expected: `All tests successful.` — 9 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Portal.pm Model/t/portal.t +git commit -m "feat(apollo): add Portal module for the UniDB organism list" +``` + +--- + +## Task 3: `ApolloRelease::Apollo` + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Apollo.pm` +- Create: `Model/t/apollo.t` +- Create: `Model/t/fixtures/apollo.json` + +- [ ] **Step 1: Create the fixture** + +Create `Model/t/fixtures/apollo.json` — a trimmed, realistic slice of +`POST /organism/findAllOrganisms`: + +```json +[ + {"id": 1484940, "commonName": "Toxoplasma gondii ME49 [Jul 01, 2023]", + "directory": "/data/apollo_data/tgonME49", + "blatdb": "/data/apollo_data/twoBit/tgonME49.2bit", + "annotationCount": 10, "publicMode": true}, + {"id": 2452162, "commonName": "Cryptococcus neoformans var. neoformans JEC21 [Jun 16, 2016]", + "directory": "/data/apollo_data/cneoJEC21", + "blatdb": "/data/apollo_data/twoBit/cneoJEC21.2bit", + "annotationCount": 14, "publicMode": true}, + {"id": 5146948, "commonName": "Candida glabrata CBS 138 [s02-m07-r27]", + "directory": "/data/apollo_data/cglaCBS138", + "blatdb": "/data/apollo_data/twoBit/cglaCBS138.2bit", + "annotationCount": 0, "publicMode": false}, + {"id": 9000001, "commonName": "Trypanosoma brucei Lister strain 427 2018 [May 01, 2019]", + "directory": "/data/apollo_data/tbruLister427_2018", + "blatdb": "/data/apollo_data/twoBit/tbruLister427_2018.2bit", + "annotationCount": 3, "publicMode": true}, + {"id": 9000002, "commonName": "Something Renamed In The Gui", + "directory": "/data/apollo_data/pfal3D7", + "blatdb": "/data/apollo_data/twoBit/pfal3D7.2bit", + "annotationCount": 3, "publicMode": true} +] +``` + +- [ ] **Step 2: Write the failing test** + +Create `Model/t/apollo.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 8; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Apollo; + +my $A = 'ApiCommonModel::Model::ApolloRelease::Apollo'; + +my $live = $A->loadFromFile("Model/t/fixtures/apollo.json"); + +is(ref($live), 'HASH', 'returns a hash keyed by abbrev'); +is(scalar(keys %$live), 5, 'all five organisms parsed'); + +my $t = $live->{tgonME49}; +is($t->{id}, 1484940, 'numeric id carried'); +is($t->{annotation_count}, 10, 'annotation count carried'); +is($t->{abbrev}, 'tgonME49', 'abbrev parsed from directory'); + +my $c = $live->{cneoJEC21}; +is($c->{annotation_count}, 14, 'the organism we must not lose is parsed'); + +# commonName is curator-editable, so it is cross-checked, never matched on. +is($A->commonNameDisagrees($live->{pfal3D7}, 'Plasmodium falciparum 3D7'), 1, + 'GUI-edited common name is flagged'); +is($A->commonNameDisagrees($live->{tgonME49}, 'Toxoplasma gondii ME49'), 0, + 'matching common name is not flagged'); +``` + +- [ ] **Step 3: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/apollo.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 4: Implement** + +Create `Model/lib/perl/ApolloRelease/Apollo.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Apollo; + +use strict; +use warnings; + +use JSON; +use LWP::UserAgent; +use HTTP::Request::Common qw(POST); + +# Reads the live prod Apollo organism roster. This is the SEED for the +# release roster -- it is the only record of what curators actually decided. +# +# Organisms are keyed by the abbrev parsed out of `directory`, never by +# commonName: `directory` is machine-written by our own update commands, +# while commonName is editable in the Apollo GUI. + +sub loadFromApi { + my ($class) = @_; + + my $url = $ENV{APOLLO_API_URL} || 'https://apollo-api.veupathdb.org'; + my $user = $ENV{APOLLO_API_USER} or die "APOLLO_API_USER is not set\n"; + my $pass = $ENV{APOLLO_API_PASS} or die "APOLLO_API_PASS is not set\n"; + + my $agent = LWP::UserAgent->new(timeout => 900); + my $response = $agent->request( + POST "$url/organism/findAllOrganisms", + Content_Type => 'form-data', + Content => [username => $user, password => $pass], + ); + + die "Apollo API request failed: " . $response->status_line . "\n" + unless $response->is_success; + + my $decoded = decode_json($response->content); + + # An empty roster would make every organism look new, and the generated + # commands would try to re-add the entire set. Fail instead. + die "Apollo returned no organisms from $url.\n" + . "Check APOLLO_API_USER/PASS, and check that you are running this on a\n" + . "Penn host -- the API is IP-restricted.\n" + unless @$decoded; + + return $class->_normalise($decoded); +} + +sub loadFromFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read $path: $!"; + local $/; + my $json = <$fh>; + close $fh; + + return $class->_normalise(decode_json($json)); +} + +sub _normalise { + my ($class, $decoded) = @_; + + my %byAbbrev; + + foreach my $raw (@$decoded) { + my $directory = $raw->{directory} || ''; + $directory =~ s{/+$}{}; + + my ($abbrev) = $directory =~ m{([^/]+)$}; + + unless ($abbrev) { + warn "Apollo organism id $raw->{id} has an unparseable directory '$raw->{directory}'; skipping\n"; + next; + } + + $byAbbrev{$abbrev} = { + abbrev => $abbrev, + id => $raw->{id}, + common_name => $raw->{commonName}, + directory => $raw->{directory}, + blatdb => $raw->{blatdb}, + annotation_count => $raw->{annotationCount} || 0, + public_mode => $raw->{publicMode} ? 1 : 0, + }; + } + + return \%byAbbrev; +} + +# Apollo names an organism " []". Compare only +# the part before the bracket. +sub commonNameDisagrees { + my ($class, $apolloOrganism, $portalName) = @_; + + my $live = $apolloOrganism->{common_name} || ''; + $live =~ s{\s*\[[^\]]*\]\s*$}{}; + + return ($live eq $portalName) ? 0 : 1; +} + +1; +``` + +- [ ] **Step 5: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/apollo.t"' +``` + +Expected: 8 passing. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Apollo.pm Model/t/apollo.t Model/t/fixtures/apollo.json +git commit -m "feat(apollo): add Apollo module for the live prod roster" +``` + +--- + +## Task 4: `ApolloRelease::Overlay` and the seeded roster file + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Overlay.pm` +- Create: `Model/data/apollo/roster-overlay.txt` +- Create: `Model/t/overlay.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/overlay.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 7; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Overlay; + +my $O = 'ApiCommonModel::Model::ApolloRelease::Overlay'; + +my $text = <<'OVERLAY'; +# comment line, ignored +remove hsapREF # host genome; Apollo curates pathogens + +add someNewOrg # approved by Uli 2026-08-21 +OVERLAY + +my $overlay = $O->parseString($text); + +is_deeply([sort keys %{$overlay->{remove}}], ['hsapREF'], 'remove parsed'); +is_deeply([sort keys %{$overlay->{add}}], ['someNewOrg'], 'add parsed'); +like($overlay->{remove}{hsapREF}, qr/host genome/, 'reason retained'); +like($overlay->{add}{someNewOrg}, qr/Uli/, 'approver retained'); + +eval { $O->parseString("bogus hsapREF # what\n") }; +like($@, qr/unknown directive 'bogus'/, 'unknown directive rejected'); + +eval { $O->parseString("add hsapREF # x\nremove hsapREF # y\n") }; +like($@, qr/both add and remove/, 'contradictory entry rejected'); + +eval { $O->parseString("add\n") }; +like($@, qr/missing organism/, 'malformed line rejected'); +``` + +A reason is *required*, and contradictions are fatal. The overlay is the only written record +of a curation decision, so a line without provenance defeats the point of having the file. + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/overlay.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Overlay.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Overlay; + +use strict; +use warnings; + +# The human decisions layered over the live Apollo roster: +# +# add # who approved it, when, why +# remove # who approved it, when, why +# +# The reason is mandatory. This file exists to record provenance; a line +# without it is indistinguishable from the hardcoded __DATA__ block this +# replaces. + +sub parseFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read roster overlay $path: $!"; + local $/; + my $text = <$fh>; + close $fh; + + return $class->parseString($text); +} + +sub parseString { + my ($class, $text) = @_; + + my %overlay = (add => {}, remove => {}); + my $lineNumber = 0; + + foreach my $line (split /\n/, $text) { + $lineNumber++; + + next if $line =~ /^\s*$/; + next if $line =~ /^\s*#/; + + my ($directive, $abbrev, $reason) = + $line =~ /^\s*(\S+)\s+(\S+)\s*#\s*(.+?)\s*$/; + + unless (defined $directive) { + my ($lone) = $line =~ /^\s*(\S+)\s*$/; + die "roster overlay line $lineNumber: missing organism or reason: '$line'\n" + if $lone; + die "roster overlay line $lineNumber: every entry needs a '# reason': '$line'\n"; + } + + die "roster overlay line $lineNumber: unknown directive '$directive' (expected add or remove)\n" + unless $directive eq 'add' || $directive eq 'remove'; + + my $other = $directive eq 'add' ? 'remove' : 'add'; + die "roster overlay line $lineNumber: $abbrev is listed as both add and remove\n" + if exists $overlay{$other}{$abbrev}; + + $overlay{$directive}{$abbrev} = $reason; + } + + return \%overlay; +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/overlay.t"' +``` + +Expected: 7 passing. + +- [ ] **Step 5: Create the seeded overlay** + +Create `Model/data/apollo/roster-overlay.txt`. Per spec §3, the 15 pre-b68 qualifying +organisms absent from Apollo are **not** one group — `hcapNAm1` is deliberately omitted from +this file so it surfaces as an add candidate: + +``` +# Roster overlay for the Apollo release package. +# +# The roster is seeded from LIVE PROD APOLLO. This file records the human +# decisions layered on top of it. Every line needs a reason: this file is the +# only written record of why an organism is or is not in Apollo. +# +# add # who approved, when, why +# remove # who approved, when, why + +# --- Host genomes ------------------------------------------------------- +# Apollo curates pathogens. Paul's script excluded these implicitly, via an +# @databases array that omitted HostDB and SchistoDB. On the UniDB portal +# that filter does not exist, so the exclusion has to be stated. + +remove hsapREF # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmusC57BL6J # host genome (seeded 2026-08-21 from b68 behaviour) +remove rnorBNNHsdMcwi # host genome (seeded 2026-08-21 from b68 behaviour) +remove btauHereford # host genome (seeded 2026-08-21 from b68 behaviour) +remove clupfamiliarisSID07034 # host genome (seeded 2026-08-21 from b68 behaviour) +remove ggalbGalGal1 # host genome (seeded 2026-08-21 from b68 behaviour) +remove cpor2N # host genome (seeded 2026-08-21 from b68 behaviour) +remove mfasREF # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmulAG07107 # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmyomMyoMyo1 # host genome (seeded 2026-08-21 from b68 behaviour) +remove dmeliso-1 # host genome (seeded 2026-08-21 from b68 behaviour) + +# --- Model fungi -------------------------------------------------------- +# In FungiDB, qualify on the criteria, never been in Apollo. Seeded to match +# existing behaviour, but nobody has consciously decided this -- raise with the +# curation team. + +remove scerS288C # model organism, never in Apollo; decision unconfirmed +remove spom972h # model organism, never in Apollo; decision unconfirmed + +# --- Deliberately NOT listed here --------------------------------------- +# hcapNAm1 (Histoplasma mississippiense NAm1) is a FungiDB pathogen that +# qualifies and is absent from Apollo. It is left in the add-candidate bucket +# for the curation team rather than suppressed here. +``` + +- [ ] **Step 6: Verify the seeded file parses** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && \ + perl -Mlib=\$GUS_HOME/lib/perl -MApiCommonModel::Model::ApolloRelease::Overlay -e ' + my \$o = ApiCommonModel::Model::ApolloRelease::Overlay->parseFile(\$ENV{GUS_HOME}.\"/data/ApiCommonModel/Model/apollo/roster-overlay.txt\"); + printf(\"removes=%d adds=%d\n\", scalar(keys %{\$o->{remove}}), scalar(keys %{\$o->{add}})); + die \"hcapNAm1 must NOT be suppressed\n\" if \$o->{remove}{hcapNAm1}; + print \"hcapNAm1 correctly left as an add candidate\n\"; + '"' +``` + +Expected: `removes=13 adds=0` and the `hcapNAm1` line. + +- [ ] **Step 7: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Overlay.pm Model/t/overlay.t Model/data/apollo/roster-overlay.txt +git commit -m "feat(apollo): add roster overlay parser and seeded overlay file" +``` + +--- + +## Task 5: `ApolloRelease::Reconcile` — buckets + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Reconcile.pm` +- Create: `Model/t/reconcile.t` + +This module is pure: hashrefs in, hashrefs out. No database, no network, no filesystem — +which is why the whole decision layer can be tested from fixtures. + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/reconcile.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 12; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Portal; +use ApiCommonModel::Model::ApolloRelease::Apollo; +use ApiCommonModel::Model::ApolloRelease::Overlay; +use ApiCommonModel::Model::ApolloRelease::Reconcile; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Reconcile'; + +my $portal = ApiCommonModel::Model::ApolloRelease::Portal->loadFromFile("Model/t/fixtures/portal.json"); +my $live = ApiCommonModel::Model::ApolloRelease::Apollo->loadFromFile("Model/t/fixtures/apollo.json"); +my $overlay = ApiCommonModel::Model::ApolloRelease::Overlay->parseString("remove hsapREF # host genome\n"); + +# No renames supplied: cneoJEC21 and cglaCBS138 are unmatched, so both are prunes. +my $r = $R->reconcile($portal, $live, $overlay, {}); + +my %update = map { $_->{abbrev} => 1 } @{$r->{update}}; +ok($update{tgonME49}, 'in Apollo and on portal -> update'); +ok($update{pfal3D7}, 'GUI-renamed organism still updates (matched on directory)'); + +my %prune = map { $_->{abbrev} => 1 } @{$r->{prune_candidate}}; +ok($prune{cneoJEC21}, 'in Apollo, absent from portal -> prune candidate'); +ok($prune{cglaCBS138}, 'second orphan is a prune candidate'); + +my ($cneo) = grep { $_->{abbrev} eq 'cneoJEC21' } @{$r->{prune_candidate}}; +is($cneo->{annotation_count}, 14, 'prune candidates carry their annotation count'); + +my %add = map { $_->{abbrev} => 1 } @{$r->{add_candidate}}; +ok($add{hcapNAm1}, 'qualifying organism absent from Apollo -> add candidate'); +ok($add{cdenJEC21}, 'rename target is an add candidate when no rename is detected'); +ok(!$add{hsapREF}, 'overlay remove suppresses an add candidate'); + +my %exception = map { $_->{abbrev} => 1 } @{$r->{exception}}; +ok($exception{tbruLister427_2018}, 'in Apollo but not reference+annotated -> exception'); +ok(!$update{tbruLister427_2018}, 'an exception is not also an update'); + +# Now supply the rename, and the pair must collapse. +my $renames = { cneoJEC21 => 'cdenJEC21' }; +my $r2 = $R->reconcile($portal, $live, $overlay, $renames); + +my %prune2 = map { $_->{abbrev} => 1 } @{$r2->{prune_candidate}}; +my %add2 = map { $_->{abbrev} => 1 } @{$r2->{add_candidate}}; +ok(!$prune2{cneoJEC21} && !$add2{cdenJEC21}, + 'a detected rename removes both the prune and the add'); + +is(scalar(@{$r2->{rename}}), 1, 'and produces exactly one rename entry'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/reconcile.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Reconcile.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Reconcile; + +use strict; +use warnings; + +use ApiCommonModel::Model::ApolloRelease::Portal; + +# Pure set logic over the three inputs. Produces five buckets: +# +# update in Apollo and on the portal -> regenerate + update +# add_candidate qualifies, not in Apollo -> needs approval +# prune_candidate in Apollo, gone from the portal -> needs approval +# rename same genome under a new abbrev -> repoint in place +# exception in Apollo, fails the criteria -> no action, reported +# +# The roster is SEEDED FROM APOLLO. reference+annotated only ever proposes an +# add; it never removes anything. + +sub reconcile { + my ($class, $portal, $live, $overlay, $renames) = @_; + + $renames ||= {}; + + my %result = ( + update => [], + add_candidate => [], + prune_candidate => [], + rename => [], + exception => [], + ); + + my %renameTarget = reverse %$renames; + + foreach my $abbrev (sort keys %$live) { + my $apollo = $live->{$abbrev}; + + if (my $newAbbrev = $renames->{$abbrev}) { + my $target = $portal->{$newAbbrev} + or die "rename target $newAbbrev is not on the portal\n"; + + push @{$result{rename}}, { + from_abbrev => $abbrev, + to_abbrev => $newAbbrev, + apollo_id => $apollo->{id}, + annotation_count => $apollo->{annotation_count}, + organism => $target, + }; + next; + } + + my $organism = $portal->{$abbrev}; + + unless ($organism) { + push @{$result{prune_candidate}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + common_name => $apollo->{common_name}, + annotation_count => $apollo->{annotation_count}, + }; + next; + } + + # In Apollo but failing the criteria: a decision made once, reported so + # nobody "fixes" it, and so a newly-demoted organism becomes visible. + unless (ApiCommonModel::Model::ApolloRelease::Portal->qualifies($organism)) { + push @{$result{exception}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + organism => $organism, + annotation_count => $apollo->{annotation_count}, + is_reference => $organism->{is_reference}, + is_annotated => $organism->{is_annotated}, + }; + next; + } + + push @{$result{update}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + organism => $organism, + }; + } + + foreach my $abbrev (sort keys %$portal) { + next if $live->{$abbrev}; + next if $renameTarget{$abbrev}; + next if $overlay->{remove}{$abbrev}; + + my $organism = $portal->{$abbrev}; + next unless ApiCommonModel::Model::ApolloRelease::Portal->qualifies($organism) + || $overlay->{add}{$abbrev}; + + push @{$result{add_candidate}}, { + abbrev => $abbrev, + organism => $organism, + approved => $overlay->{add}{$abbrev} ? 1 : 0, + reason => $overlay->{add}{$abbrev}, + }; + } + + $class->assertInvariants(\%result); + + return \%result; +} + +# Invariant 4 from the spec. An add plus a prune for the same genome creates a +# new Apollo organism with no annotations and orphans the one holding the +# curation work -- from the same inputs and the same API as the correct +# behaviour. This is the failure mode that must never ship. +sub assertInvariants { + my ($class, $result) = @_; + + my %renamedFrom = map { $_->{from_abbrev} => 1 } @{$result->{rename}}; + my %renamedTo = map { $_->{to_abbrev} => 1 } @{$result->{rename}}; + + foreach my $add (@{$result->{add_candidate}}) { + die "INVARIANT VIOLATED: $add->{abbrev} is both a rename target and an add candidate\n" + if $renamedTo{$add->{abbrev}}; + } + + foreach my $prune (@{$result->{prune_candidate}}) { + die "INVARIANT VIOLATED: $prune->{abbrev} is both a rename source and a prune candidate\n" + if $renamedFrom{$prune->{abbrev}}; + } + + return 1; +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/reconcile.t"' +``` + +Expected: 12 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Reconcile.pm Model/t/reconcile.t +git commit -m "feat(apollo): add reconciliation with add/prune/rename invariant" +``` + +--- + +## Task 6: `ApolloRelease::Rename` — detection by sequence identity + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Rename.pm` +- Create: `Model/t/rename.t` + +Species taxon ID must **not** be used: it changed in both known renames +(`cneoJEC21` 5207 → `cdenJEC21` 40410). Strain abbrev narrows the candidates; identical +sequence names and lengths decide. + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/rename.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 6; +use File::Temp qw(tempdir); +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Rename; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Rename'; + +my $dir = tempdir(CLEANUP => 1); + +sub writeFai { + my ($path, @lines) = @_; + open(my $fh, '>', $path) or die $!; + print $fh "$_\n" for @lines; + close $fh; +} + +# Old organism's shipped index, and a new organism with the same assembly. +writeFai("$dir/old.fai", "AE017341.1\t2300533\t60\t60\t61", "AE017342.1\t1632307\t60\t60\t61"); +writeFai("$dir/same.fai", "AE017341.1\t2300533\t99\t70\t71", "AE017342.1\t1632307\t99\t70\t71"); +writeFai("$dir/other.fai", "CP000001.1\t123456\t60\t60\t61"); + +is_deeply($R->readFai("$dir/old.fai"), + {'AE017341.1' => 2300533, 'AE017342.1' => 1632307}, + 'fai parsed to name => length, ignoring offset columns'); + +ok($R->sameAssembly("$dir/old.fai", "$dir/same.fai"), + 'identical names and lengths match despite different byte offsets'); +ok(!$R->sameAssembly("$dir/old.fai", "$dir/other.fai"), + 'different assembly does not match'); + +my $portal = { + cdenJEC21 => {abbrev => 'cdenJEC21', strain_abbrev => 'JEC21', name_for_filenames => 'CdeneoformansJEC21'}, + tgonME49 => {abbrev => 'tgonME49', strain_abbrev => 'ME49', name_for_filenames => 'TgondiiME49'}, +}; + +my $renames = $R->detect( + ['cneoJEC21'], + $portal, + sub { my ($abbrev) = @_; return "$dir/old.fai" }, # previous release + sub { my ($org) = @_; return $org->{abbrev} eq 'cdenJEC21' ? "$dir/same.fai" : "$dir/other.fai" }, + {cneoJEC21 => 'JEC21'}, +); + +is_deeply($renames, {cneoJEC21 => 'cdenJEC21'}, 'rename detected by sequence identity'); + +# A missing previous-release index must not silently mean "no rename". +my $none = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/does-not-exist.fai" }, + sub { return "$dir/same.fai" }, + {cneoJEC21 => 'JEC21'}); +is_deeply($none, {}, 'unresolvable organism yields no rename'); + +my @warnings = $R->warnings(); +like($warnings[0], qr/cneoJEC21/, 'and says so rather than staying silent'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/rename.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Rename.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Rename; + +use strict; +use warnings; + +# Detects taxonomic renames: the same assembly appearing under a new organism +# abbrev. Species taxon ID is deliberately NOT used -- it changed in both +# known cases (cneoJEC21 5207 -> cdenJEC21 40410). +# +# Strain abbrev narrows the candidate set; identical sequence names and +# lengths decide. + +my @WARNINGS; + +sub warnings { return @WARNINGS } + +sub readFai { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or return undef; + + my %lengths; + while (my $line = <$fh>) { + chomp $line; + next unless length $line; + my ($name, $length) = split /\t/, $line; + $lengths{$name} = $length + 0; + } + close $fh; + + return \%lengths; +} + +sub sameAssembly { + my ($class, $pathA, $pathB) = @_; + + my $a = $class->readFai($pathA) or return 0; + my $b = $class->readFai($pathB) or return 0; + + return 0 unless scalar(keys %$a) == scalar(keys %$b); + return 0 unless scalar(keys %$a); + + foreach my $name (keys %$a) { + return 0 unless defined $b->{$name}; + return 0 unless $b->{$name} == $a->{$name}; + } + + return 1; +} + +# $orphans - arrayref of Apollo abbrevs absent from the portal +# $portal - the portal hash from Portal.pm +# $previousFai - coderef: abbrev -> path to last release's .fa.fai +# $currentFai - coderef: portal organism -> path to this build's .fai +# $strainByAbbrev - hashref: orphan abbrev -> its strain abbrev +sub detect { + my ($class, $orphans, $portal, $previousFai, $currentFai, $strainByAbbrev) = @_; + + @WARNINGS = (); + my %renames; + + foreach my $orphan (@$orphans) { + my $oldFai = $previousFai->($orphan); + + unless ($oldFai && -e $oldFai) { + push @WARNINGS, + "$orphan: no index from the previous release at " + . ($oldFai || '(no path)') + . "; cannot test for a rename. Treating as a prune candidate."; + next; + } + + my $strain = $strainByAbbrev->{$orphan}; + + my @candidates = grep { + !defined($strain) || (defined($_->{strain_abbrev}) && $_->{strain_abbrev} eq $strain) + } values %$portal; + + my @matches; + foreach my $candidate (@candidates) { + next if $candidate->{abbrev} eq $orphan; + my $newFai = $currentFai->($candidate); + next unless $newFai && -e $newFai; + push @matches, $candidate->{abbrev} + if $class->sameAssembly($oldFai, $newFai); + } + + if (@matches == 1) { + $renames{$orphan} = $matches[0]; + } + elsif (@matches > 1) { + push @WARNINGS, + "$orphan: matches more than one portal organism (" + . join(', ', sort @matches) + . "); refusing to guess. Treating as a prune candidate."; + } + else { + push @WARNINGS, + "$orphan: no portal organism shares its assembly. Treating as a prune candidate."; + } + } + + return \%renames; +} + +1; +``` + +An ambiguous match produces **no** rename. Guessing here repoints curated annotations at the +wrong genome. + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/rename.t"' +``` + +Expected: 6 passing. + +- [ ] **Step 5: Verify against the real rename** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && \ + perl -Mlib=\$GUS_HOME/lib/perl -MApiCommonModel::Model::ApolloRelease::Rename -e '\'' + my \$R = q(ApiCommonModel::Model::ApolloRelease::Rename); + my \$W = q(/var/www/Common/apiSiteFilesMirror/webServices/UniDB/build-71); + print \"cdenJEC21 matches: \", \$R->sameAssembly(q(/tmp/cneoJEC21.fa.fai), \"\$W/CdeneoformansJEC21/genomeAndProteome/fasta/genome.fasta.fai\") ? \"YES\" : \"no\", \"\n\"; + print \"tgonME49 matches: \", \$R->sameAssembly(q(/tmp/cneoJEC21.fa.fai), \"\$W/TgondiiME49/genomeAndProteome/fasta/genome.fasta.fai\") ? \"YES\" : \"no\", \"\n\"; + '\''"' +``` + +First copy the b68 index across: `scp yew:/eupath/data/apolloConfigs/release-68/prod/data/cneoJEC21/seq/cneoJEC21.fa.fai /tmp/` on cedar. + +Expected: `cdenJEC21 matches: YES`, `tgonME49 matches: no`. + +- [ ] **Step 6: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Rename.pm Model/t/rename.t +git commit -m "feat(apollo): detect taxonomic renames by assembly identity + +Species taxon ID cannot be used: it changed in both known renames. +Ambiguous matches produce no rename rather than a guess." +``` + +--- + +## Task 7: `ApolloRelease::Absolutize` + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Absolutize.pm` +- Create: `Model/t/absolutize.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/absolutize.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 7; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Absolutize; + +my $A = 'ApiCommonModel::Model::ApolloRelease::Absolutize'; +my $BASE = 'https://veupathdb.org'; + +is($A->rewrite('"urlTemplate":"/a/service/jbrowse/store?data=x"', $BASE), + '"urlTemplate":"https://veupathdb.org/a/service/jbrowse/store?data=x"', + 'store URL in JSON'); + +is($A->rewrite("", $BASE), + "", + 'URL inside an HTML blob'); + +is($A->rewrite("function(t,f) { return '/a/app/record/gene/' + f.get('name') }", $BASE), + "function(t,f) { return 'https://veupathdb.org/a/app/record/gene/' + f.get('name') }", + 'URL inside a JavaScript function body'); + +is($A->rewrite('"baseUrl":"/a/service/jbrowse"', $BASE), + '"baseUrl":"https://veupathdb.org/a/service/jbrowse"', + 'bare service base with no trailing path'); + +is($A->rewrite('already https://veupathdb.org/a/x', $BASE), + 'already https://veupathdb.org/a/x', + 'an already-absolute URL is not rewritten twice'); + +is($A->rewrite('a path like /data/a/thing', $BASE), + 'a path like /data/a/thing', + 'a mid-path /a/ that is not a site-root URL is left alone'); + +eval { $A->assertNoRelative('{"url":"/a/service/x"}', 'trackList.json') }; +like($@, qr/trackList\.json/, 'the assertion names the offending file'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/absolutize.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Absolutize.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Absolutize; + +use strict; +use warnings; + +# Apollo needs absolute URLs; the jbrowse scripts emit site-relative "/a/...". +# +# This is a text pass rather than a typed transformation because roughly 25 of +# these live inside free text -- HTML blobs, onClick and menuTemplate URLs, and +# JavaScript function bodies -- where a typed getApolloObject() cannot reach. +# See spec section 6. +# +# The post-condition check is not optional. Paul's version did the same +# rewrite with no verification, so a missed URL became a track that silently +# 404s inside Apollo. + +# A site-root URL is "/a/" appearing at the start of a quoted string or +# immediately after a quote/paren/equals/whitespace -- not "/a/" occurring in +# the middle of a longer path such as /data/a/thing. +my $RELATIVE = qr{(?<=['"(=\s])/a/}; + +sub rewrite { + my ($class, $text, $base) = @_; + + return $text unless defined $text; + + $base =~ s{/+$}{}; + $text =~ s{$RELATIVE}{$base/a/}g; + + return $text; +} + +sub assertNoRelative { + my ($class, $text, $label) = @_; + + return 1 unless defined $text; + + my @found; + while ($text =~ m{$RELATIVE}g) { + my $start = $-[0]; + push @found, substr($text, $start, 60); + last if @found >= 3; + } + + die "$label still contains site-relative URLs after absolutization:\n" + . join('', map { " $_\n" } @found) + if @found; + + return 1; +} + +1; +``` + +The lookbehind is what makes this safe: it anchors on a delimiter, so a genuine path +component like `/data/a/thing` is untouched, and an already-absolute +`https://veupathdb.org/a/x` is not rewritten twice because the character before `/a/` is `g`. + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/absolutize.t"' +``` + +Expected: 7 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Absolutize.pm Model/t/absolutize.t +git commit -m "feat(apollo): add URL absolutization with a mandatory post-condition check" +``` + +--- + +## Task 8: `ApolloRelease::Generate` — per-organism files + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Generate.pm` +- Create: `Model/t/generate.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/generate.t`. This tests the parts that do not need a database — the +`trackList.json` assembly and per-organism failure isolation — by injecting a fake runner: + +```perl +use strict; +use warnings; +use Test::More tests => 8; +use File::Temp qw(tempdir); +use JSON; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Generate; + +my $G = 'ApiCommonModel::Model::ApolloRelease::Generate'; + +my $trackList = { + refSeqs => "/a/service/jbrowse/refSeqs/tgonME49", + names => {url => "/a/service/jbrowse/names/tgonME49"}, + include => [ + "/a/jbrowse/tracks/tgonME49/tracks.conf", + "/a/service/jbrowse/rnaseqJunctions/tgonME49", + "/a/jbrowse/user-datasets-jbrowse/tgonME49", + "/a/jbrowse/functions.conf", + ], + tracks => [{label => "should be replaced"}], +}; + +my $built = $G->buildTrackList($trackList, 'tgonME49', 'https://veupathdb.org'); + +is_deeply([grep { /user-datasets/ } @{$built->{include}}], [], + 'user dataset includes are dropped'); +is_deeply($built->{include}, + ['tracks.conf', 'rnaseqJunctions.json', 'functions.conf'], + 'includes rewritten to local filenames'); +is($built->{refSeqs}, 'seq/tgonME49.fa.fai', 'refSeqs points at the local index'); +is(scalar(@{$built->{tracks}}), 1, 'exactly one track remains'); +is($built->{tracks}[0]{storeClass}, 'JBrowse/Store/SeqFeature/IndexedFasta', + 'and it is the local reference sequence track'); +is($built->{tracks}[0]{urlTemplate}, 'seq/tgonME49.fa', 'pointing at the local fasta'); +like($built->{names}{url}, qr{^https://veupathdb\.org/a/}, 'names url absolutized'); + +# One organism failing must not abort the run. +my $dir = tempdir(CLEANUP => 1); +my $results = $G->generateAll( + [{abbrev => 'good1'}, {abbrev => 'explodes'}, {abbrev => 'good2'}], + $dir, + sub { + my ($organism) = @_; + die "simulated failure\n" if $organism->{abbrev} eq 'explodes'; + return 1; + }, +); +is_deeply([sort @{$results->{failed}}], ['explodes'], + 'a failing organism is recorded and the run continues'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/generate.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Generate.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Generate; + +use strict; +use warnings; + +use JSON; +use File::Path qw(make_path); +use File::Copy qw(cp); + +use ApiCommonModel::Model::ApolloRelease::Absolutize; + +my $ABS = 'ApiCommonModel::Model::ApolloRelease::Absolutize'; + +# Maps an include URL to the local filename Apollo will read it as. +my @INCLUDE_NAMES = ( + [qr{tracks\.conf$} => 'tracks.conf'], + [qr{functions\.conf$} => 'functions.conf'], + [qr{bindingSites\.conf$} => 'bindingSites.conf'], + [qr{apollo_gene_tracks\.conf$} => 'apollo_gene_tracks.conf'], + [qr{rnaseqJunctions} => 'rnaseqJunctions.json'], + [qr{organismSpecific} => 'organismSpecific.json'], + [qr{rnaseq}i => 'rnaseq.json'], + [qr{chipseq}i => 'chipseq.json'], + [qr{dnaseq}i => 'dnaseq.json'], +); + +sub localNameForInclude { + my ($class, $url) = @_; + + foreach my $rule (@INCLUDE_NAMES) { + my ($pattern, $name) = @$rule; + return $name if $url =~ $pattern; + } + + return undef; +} + +# Rewrites the site's trackList.json into Apollo's. Three deliberate +# differences, all declared in spec section 10: +# - user dataset includes dropped +# - includes rewritten to local filenames +# - the reference sequence track replaced with a local IndexedFasta track +sub buildTrackList { + my ($class, $trackList, $abbrev, $base) = @_; + + my %built = %$trackList; + + my @includes; + foreach my $url (@{$trackList->{include} || []}) { + next if $url =~ m{user-datasets-jbrowse}; + my $name = $class->localNameForInclude($url); + unless ($name) { + warn "$abbrev: unrecognised include '$url'; skipping\n"; + next; + } + push @includes, $name; + } + $built{include} = \@includes; + + $built{refSeqs} = "seq/$abbrev.fa.fai"; + + $built{names} = {%{$trackList->{names} || {}}}; + $built{names}{url} = $ABS->rewrite($built{names}{url}, $base) + if defined $built{names}{url}; + + $built{tracks} = [ + { + category => "Sequence Analysis", + faiUrlTemplate => "seq/$abbrev.fa.fai", + key => "Reference sequence", + label => "DNA", + seqType => "dna", + storeClass => "JBrowse/Store/SeqFeature/IndexedFasta", + type => "SequenceTrack", + urlTemplate => "seq/$abbrev.fa", + useAsRefSeqStore => JSON::true, + } + ]; + + return \%built; +} + +# Runs $perOrganism for each organism, isolating failures. Paul's script died +# on the first bad organism, discarding hours of completed work; this run is +# long and the organisms are independent. +sub generateAll { + my ($class, $organisms, $outDir, $perOrganism) = @_; + + my %results = (succeeded => [], failed => [], errors => {}); + + foreach my $organism (@$organisms) { + my $abbrev = $organism->{abbrev}; + + my $ok = eval { $perOrganism->($organism); 1 }; + + if ($ok) { + push @{$results{succeeded}}, $abbrev; + } + else { + my $error = $@ || 'unknown error'; + chomp $error; + push @{$results{failed}}, $abbrev; + $results{errors}{$abbrev} = $error; + warn "FAILED $abbrev: $error\n"; + } + } + + return \%results; +} + +sub writeFile { + my ($class, $path, $content, $base) = @_; + + my $rewritten = $ABS->rewrite($content, $base); + $ABS->assertNoRelative($rewritten, $path); + + open(my $fh, '>', $path) or die "Cannot write $path: $!"; + print $fh $rewritten; + close $fh; + + return 1; +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/generate.t"' +``` + +Expected: 8 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Generate.pm Model/t/generate.t +git commit -m "feat(apollo): add per-organism generation with failure isolation" +``` + +--- + +## Task 9: Probe one real organism end to end + +Before wiring the CLI, confirm the `jbrowse*` scripts actually produce usable Apollo output. +This is the open risk in the spec: the track layer moved REST → flat files and Oracle → +Postgres since the last successful run, so nothing about this is assumed. + +**Files:** +- Create: `Model/bin/apolloProbeOrganism` (a temporary harness, deleted in Task 13) + +- [ ] **Step 1: Run each producing script by hand for `tgonME49`** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && \ + W=/var/www/Common/apiSiteFilesMirror/webServices && \ + for spec in \ + \"jbrowseRnaAndChipSeqTracks tgonME49 UniDB 71 \$W RNASeq jbrowse\" \ + \"jbrowseRnaAndChipSeqTracks tgonME49 UniDB 71 \$W ChIPSeq jbrowse\" \ + \"jbrowseRNASeqJunctionTracks tgonME49 UniDB 71 \$W 1 jbrowse\" \ + \"jbrowseOrganismSpecificTracks tgonME49 UniDB 1 71 \$W jbrowse\" \ + \"jbrowseDNASeqTracks tgonME49 UniDB 71 \$W jbrowse\" ; do \ + name=\$(echo \$spec | cut -d\" \" -f1); \ + out=/tmp/probe_\$name.json; err=/tmp/probe_\$name.err; \ + perl \$GUS_HOME/bin/\$spec > \$out 2> \$err; \ + echo \"\$name rc=\$? out=\$(stat -c%s \$out) err=\$(stat -c%s \$err) tracks=\$(python3 -c \"import json;print(len(json.load(open(chr(39)+\$out.__str__()+chr(39)))[chr(39)+chr(116)+chr(114)+chr(97)+chr(99)+chr(107)+chr(115)+chr(39)]))\" 2>/dev/null || echo PARSE_FAIL)\"; \ + done"' +``` + +If the nested quoting fights you, write the loop into a file on cedar and run it — the point +is to record, for each script: exit code, stdout size, **stderr size**, and track count. + +- [ ] **Step 2: Record the result in the spec** + +Expected per script: `rc=0`, `err=0`, `tracks` > 0. + +Anything else is a finding that changes the plan. Specifically: +- **stderr non-empty** — the same class of bug as Task 1. These scripts are also served + through `responseFromCommand`, so warnings corrupt output. +- **`tracks=0` but valid JSON** — the characteristic flat-file migration failure. Valid, + empty, and silent. +- **A script dies** — investigate before proceeding; do not work around it in the tool. + +Append a short table of the measured results to spec §10 and commit it, so the next person +knows what "working" looked like on 2026-08-21. + +- [ ] **Step 3: Compare against what the site serves** + +```bash +# From an authenticated browser tab on the eupathdb instance: +# fetch('/a/service/jbrowse/tracks/tgonME49/trackList.json').then(r=>r.json()) +# Record: the include list, and the track count. +``` + +The Apollo package's include list must be the same set minus `user-datasets-jbrowse`, and +its track count must match after the reference-sequence substitution. A difference outside +the four declared classes in spec §10 is a defect in the tool, not in the site. + +- [ ] **Step 4: Commit the findings** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +git commit -m "docs(apollo): record measured per-script output for tgonME49" +``` + +--- + +## Task 10: `ApolloRelease::Commands` + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Commands.pm` +- Create: `Model/t/commands.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/commands.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 8; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Commands; + +my $C = 'ApiCommonModel::Model::ApolloRelease::Commands'; + +my $update = $C->updateCommand({ + abbrev => 'tgonME49', apollo_id => 1484940, +}); +like($update, qr/"id":"1484940"/, 'update targets the numeric id'); +like($update, qr{/data/apollo_data/tgonME49}, 'points at the organism directory'); +like($update, qr{/data/apollo_data/twoBit/tgonME49\.2bit}, 'and at its blatdb'); +unlike($update, qr/GFERsVNiX5BQ|password":"[^\$]/, 'no literal password in output'); + +my $rename = $C->renameCommand({ + from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', apollo_id => 2452162, + organism => {name => 'Cryptococcus deneoformans JEC21', latest_annotation_version => 'Jun 16, 2016'}, +}); +like($rename, qr/2452162/, 'rename targets the EXISTING apollo id'); +like($rename, qr{/data/apollo_data/cdenJEC21}, 'repointed at the new directory'); +like($rename, qr/Cryptococcus deneoformans JEC21 \[Jun 16, 2016\]/, 'renamed with version'); + +my $prune = $C->pruneCommand({abbrev => 'cglaCBS138', apollo_id => 5146948}); +like($prune, qr/"publicMode":"false"/, 'prune unpublishes rather than deleting'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/commands.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Commands.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Commands; + +use strict; +use warnings; + +# Emits the commands a human runs after the package is synced. This module +# never executes anything. +# +# Passwords are emitted as the shell variable $APOLLO_ADMIN_PASSWORD, never +# interpolated -- the generated files are read by several people and land in a +# shared directory. + +my $API = 'https://apollo-api.veupathdb.org'; +my $DATA = '/data/apollo_data'; +my $ADMIN = 'admin@local.host'; + +sub _updateOrganismInfo { + my ($class, $id, $abbrev, $extra) = @_; + + my %fields = ( + username => $ADMIN, + password => '$APOLLO_ADMIN_PASSWORD', + id => "$id", + directory => "$DATA/$abbrev", + blatdb => "$DATA/twoBit/$abbrev.2bit", + %{$extra || {}}, + ); + + my $json = join(',', map { "\"$_\":\"$fields{$_}\"" } + qw(username password id directory blatdb), + grep { !/^(username|password|id|directory|blatdb)$/ } sort keys %fields); + + return qq{curl -X POST -H "Content-Type: application/json" --data '{$json}' $API/organism/updateOrganismInfo\n}; +} + +sub updateCommand { + my ($class, $entry) = @_; + return $class->_updateOrganismInfo($entry->{apollo_id}, $entry->{abbrev}, + {publicMode => 'true'}); +} + +# A rename repoints the EXISTING organism, preserving its annotations. The +# alternative -- add the new abbrev, prune the old -- creates an empty organism +# and orphans the curation work. +sub renameCommand { + my ($class, $entry) = @_; + + my $organism = $entry->{organism}; + my $version = $organism->{latest_annotation_version}; + my $name = defined $version + ? "$organism->{name} [$version]" + : $organism->{name}; + + return $class->_updateOrganismInfo($entry->{apollo_id}, $entry->{to_abbrev}, + {publicMode => 'true', commonName => $name}); +} + +# Prune unpublishes. It is reversible, and Apollo's API has no delete in use. +sub pruneCommand { + my ($class, $entry) = @_; + return $class->_updateOrganismInfo($entry->{apollo_id}, $entry->{abbrev}, + {publicMode => 'false'}); +} + +sub addCommand { + my ($class, $entry) = @_; + + my $organism = $entry->{organism}; + my $version = $organism->{latest_annotation_version}; + my $name = defined $version + ? "$organism->{name} [$version]" + : $organism->{name}; + my $abbrev = $entry->{abbrev}; + + return + qq{groovy add_organism.groovy -name '$name' -url https://apollo.apidb.org } + . qq{-directory '$DATA/$abbrev' -blatdb '$DATA/twoBit/$abbrev.2bit' } + . qq{-username '$ADMIN' -password \$APOLLO_ADMIN_PASSWORD\n} + . qq{groovy alter_group_permissions.groovy -groupname remote_users -organism '$name' } + . qq{-permission WRITE -destinationurl https://apollo.apidb.org/ } + . qq{-adminusername '$ADMIN' -adminpassword \$APOLLO_ADMIN_PASSWORD\n}; +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/commands.t"' +``` + +Expected: 8 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Commands.pm Model/t/commands.t +git commit -m "feat(apollo): emit update/rename/prune/add commands + +Prune unpublishes rather than deletes. Rename repoints the existing +organism id so its annotations survive." +``` + +--- + +## Task 11: `ApolloRelease::Report` + +**Files:** +- Create: `Model/lib/perl/ApolloRelease/Report.pm` +- Create: `Model/t/report.t` + +- [ ] **Step 1: Write the failing test** + +Create `Model/t/report.t`: + +```perl +use strict; +use warnings; +use Test::More tests => 5; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Report; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Report'; + +my $result = { + update => [ {abbrev => 'tgonME49'} ], + add_candidate => [ {abbrev => 'hcapNAm1', organism => {name => 'Histoplasma mississippiense NAm1'}, approved => 0} ], + prune_candidate => [ {abbrev => 'cglaCBS138', annotation_count => 0, common_name => 'Candida glabrata CBS 138'}, + {abbrev => 'zzzRisky', annotation_count => 14, common_name => 'Something Curated'} ], + rename => [ {from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', annotation_count => 14} ], + exception => [ {abbrev => 'tbruLister427_2018', is_reference => 0, is_annotated => 1} ], +}; + +my $text = $R->render($result, {build => 71, environment => 'prod'}); + +like($text, qr/update\s+1/, 'counts each bucket'); +like($text, qr/cneoJEC21.*cdenJEC21/, 'names both sides of a rename'); +like($text, qr/zzzRisky.*14/, 'prune candidates show their annotation count'); +like($text, qr/ANNOTATIONS WILL BE HIDDEN/, 'an annotated prune is called out loudly'); +unlike($text, qr/cglaCBS138.*ANNOTATIONS WILL BE HIDDEN/, 'a zero-annotation prune is not'); +``` + +- [ ] **Step 2: Run it and watch it fail** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME/ApiCommonModel && prove -v Model/t/report.t"' +``` + +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +Create `Model/lib/perl/ApolloRelease/Report.pm`: + +```perl +package ApiCommonModel::Model::ApolloRelease::Report; + +use strict; +use warnings; + +# The artifact that goes to the curation team. Adds and prunes are proposals; +# a human decides. An annotated prune is called out because that is the +# expensive mistake. + +sub render { + my ($class, $result, $context) = @_; + + my @out; + + push @out, "Apollo release reconciliation\n"; + push @out, sprintf("build %s, environment %s\n\n", + $context->{build} // '?', $context->{environment} // '?'); + + push @out, "summary\n"; + foreach my $bucket (qw(update add_candidate prune_candidate rename exception)) { + push @out, sprintf(" %-18s %d\n", $bucket, scalar @{$result->{$bucket} || []}); + } + push @out, "\n"; + + if (@{$result->{rename} || []}) { + push @out, "renames (repointed in place; annotations preserved)\n"; + foreach my $r (sort { $a->{from_abbrev} cmp $b->{from_abbrev} } @{$result->{rename}}) { + push @out, sprintf(" %-26s -> %-26s (%d annotations)\n", + $r->{from_abbrev}, $r->{to_abbrev}, $r->{annotation_count} || 0); + } + push @out, "\n"; + } + + if (@{$result->{prune_candidate} || []}) { + push @out, "prune candidates -- REQUIRE APPROVAL\n"; + foreach my $p (sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{prune_candidate}}) { + my $count = $p->{annotation_count} || 0; + push @out, sprintf(" %-26s %3d annotations %s%s\n", + $p->{abbrev}, $count, $p->{common_name} || '', + $count ? " <-- ANNOTATIONS WILL BE HIDDEN" : ''); + } + push @out, "\n"; + } + + if (@{$result->{add_candidate} || []}) { + push @out, "add candidates -- REQUIRE APPROVAL\n"; + foreach my $a (sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{add_candidate}}) { + push @out, sprintf(" %-26s %s%s\n", $a->{abbrev}, + $a->{organism}{name} || '', + $a->{approved} ? ' (approved in overlay)' : ''); + } + push @out, "\n"; + } + + if (@{$result->{exception} || []}) { + push @out, "exceptions -- in Apollo but not reference+annotated; no action\n"; + foreach my $e (sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{exception}}) { + push @out, sprintf(" %-26s reference=%d annotated=%d\n", + $e->{abbrev}, $e->{is_reference}, $e->{is_annotated}); + } + push @out, "\n"; + } + + return join('', @out); +} + +1; +``` + +- [ ] **Step 4: Install, run, verify it passes** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/report.t"' +``` + +Expected: 5 passing. + +- [ ] **Step 5: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/lib/perl/ApolloRelease/Report.pm Model/t/report.t +git commit -m "feat(apollo): render the reconciliation report" +``` + +--- + +## Task 12: The CLI + +**Files:** +- Create: `Model/bin/createApolloReleasePackage` + +- [ ] **Step 1: Write the script** + +```perl +#!/usr/bin/perl + +use strict; +use warnings; + +use lib $ENV{GUS_HOME} . "/lib/perl"; + +use Getopt::Long; +use File::Path qw(make_path); + +use ApiCommonModel::Model::ApolloRelease::Portal; +use ApiCommonModel::Model::ApolloRelease::Apollo; +use ApiCommonModel::Model::ApolloRelease::Overlay; +use ApiCommonModel::Model::ApolloRelease::Rename; +use ApiCommonModel::Model::ApolloRelease::Reconcile; +use ApiCommonModel::Model::ApolloRelease::Report; +use ApiCommonModel::Model::ApolloRelease::Commands; + +my ($report, $generate, $help); +my $build = undef; +my $environment = 'prod'; +my $project = 'UniDB'; +my $outDir = "$ENV{HOME}/apolloConfigs"; +my $base = 'https://veupathdb.org'; +my $webServices = '/var/www/Common/apiSiteFilesMirror/webServices'; +my $previousRelease; +my @onlyOrganisms; + +GetOptions( + 'report' => \$report, + 'generate' => \$generate, + 'build=i' => \$build, + 'environment=s' => \$environment, + 'project=s' => \$project, + 'out-dir=s' => \$outDir, + 'base-url=s' => \$base, + 'webservices-dir=s' => \$webServices, + 'previous-release=s'=> \$previousRelease, + 'organism=s' => \@onlyOrganisms, + 'help|h' => \$help, +) or usage(1); + +usage(0) if $help; + +usage(1, "exactly one of --report or --generate is required") + unless ($report xor $generate); +usage(1, "--build is required") unless $build; +usage(1, "--environment must be qa or prod") + unless $environment eq 'qa' || $environment eq 'prod'; + +preflight(); + +my $portal = ApiCommonModel::Model::ApolloRelease::Portal->loadFromCommand($project); +my $live = ApiCommonModel::Model::ApolloRelease::Apollo->loadFromApi(); +my $overlay = ApiCommonModel::Model::ApolloRelease::Overlay->parseFile( + "$ENV{GUS_HOME}/data/ApiCommonModel/Model/apollo/roster-overlay.txt"); + +my @orphans = grep { !$portal->{$_} } sort keys %$live; + +my $renames = ApiCommonModel::Model::ApolloRelease::Rename->detect( + \@orphans, + $portal, + sub { + my ($abbrev) = @_; + return undef unless $previousRelease; + return "$previousRelease/data/$abbrev/seq/$abbrev.fa.fai"; + }, + sub { + my ($organism) = @_; + return "$webServices/$project/build-$build/$organism->{name_for_filenames}" + . "/genomeAndProteome/fasta/genome.fasta.fai"; + }, + { map { $_ => ($live->{$_}{common_name} =~ /(\S+)\s*\[/ ? $1 : undef) } @orphans }, +); + +warn "$_\n" for ApiCommonModel::Model::ApolloRelease::Rename->warnings(); + +my $result = ApiCommonModel::Model::ApolloRelease::Reconcile->reconcile( + $portal, $live, $overlay, $renames); + +my $text = ApiCommonModel::Model::ApolloRelease::Report->render( + $result, {build => $build, environment => $environment}); + +print $text; + +exit 0 if $report; + +my $releaseDir = "$outDir/release-$build/$environment"; +make_path("$releaseDir/data", "$releaseDir/twoBit", "$releaseDir/updateCommands"); + +open(my $rfh, '>', "$releaseDir/report.txt") or die "Cannot write report: $!"; +print $rfh $text; +close $rfh; + +# Generation is wired in Task 13; the CLI up to here is the reporting path. +die "--generate is not yet implemented; see Task 13 of the plan\n"; + +sub preflight { + foreach my $var (qw(GUS_HOME APOLLO_API_USER APOLLO_API_PASS)) { + die "$var is not set. Source the site's etc/setenv and export the Apollo credentials.\n" + unless $ENV{$var}; + } + + system("which faToTwoBit > /dev/null 2>&1") == 0 + or die "faToTwoBit is not on PATH.\n" + . "Install UCSC's linux.x86_64 build into ~/bin:\n" + . " curl -fLO https://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit\n"; +} + +sub usage { + my ($status, $message) = @_; + print STDERR "ERROR: $message\n\n" if $message; + print STDERR <<'USAGE'; +Usage: + createApolloReleasePackage --report --build NN [options] + createApolloReleasePackage --generate --build NN [options] + + --report reconcile and print the five buckets; changes nothing + --generate build the package for the approved roster + --build NN release/build number (required) + --environment qa|prod default prod + --project NAME WDK model name, default UniDB + --out-dir DIR default $HOME/apolloConfigs + --base-url URL default https://veupathdb.org + --webservices-dir DIR default /var/www/Common/apiSiteFilesMirror/webServices + --previous-release DIR previous release/ dir, used for rename detection + --organism ABBREV restrict generation to this organism; repeatable. + Reconciliation still runs over everything, so the + report and the invariants are unaffected. + +Environment: GUS_HOME, APOLLO_API_USER, APOLLO_API_PASS. +USAGE + exit $status; +} +``` + +- [ ] **Step 2: Install and run the report against live data** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && \ + export APOLLO_API_USER=api@local.host APOLLO_API_PASS=\$(read -rsp \"apollo api pass: \" P; echo \$P) && \ + perl \$GUS_HOME/bin/createApolloReleasePackage --report --build 71"' +``` + +Expected, matching the spec's measured numbers: `update 457`, `add_candidate` 35 (48 minus +the 13 seeded removes), `prune_candidate 2`, `rename 0` (no `--previous-release` given), +`exception 4`. + +Then re-run with `--previous-release`, after copying release-68 from yew to cedar: + +```bash +rsync -a yew:/eupath/data/apolloConfigs/release-68/prod/data/ ~/apolloConfigs/release-68/prod/data/ +``` + +Expected with it: `rename 1` (`cneoJEC21 -> cdenJEC21`), `prune_candidate 1` +(`cglaCBS138` only), and `cdenJEC21` gone from the add candidates. + +- [ ] **Step 3: Commit** + +```bash +cd ~/workspaces/eupathdb/ApiCommonModel +git add Model/bin/createApolloReleasePackage +git commit -m "feat(apollo): add createApolloReleasePackage CLI with --report" +``` + +--- + +## Task 13: Wire generation into the CLI + +**Files:** +- Modify: `Model/bin/createApolloReleasePackage` (replace the `die` from Task 12) +- Delete: `Model/bin/apolloProbeOrganism` + +- [ ] **Step 1: Replace the placeholder with the generation pass** + +Replace the `die "--generate is not yet implemented..."` line with: + +```perl +my @roster = ( + @{$result->{update}}, + @{$result->{rename}}, + (grep { $_->{approved} } @{$result->{add_candidate}}), +); + +# --organism narrows generation only. Reconciliation above ran over the full +# set, so the report and the add/prune/rename invariants are unaffected by it. +if (@onlyOrganisms) { + my %wanted = map { $_ => 1 } @onlyOrganisms; + @roster = grep { + my $abbrev = $_->{to_abbrev} || $_->{abbrev}; + $wanted{$abbrev}; + } @roster; + die "--organism matched nothing in the roster\n" unless @roster; + warn "restricted to " . scalar(@roster) . " organism(s) by --organism\n"; +} + +my $results = ApiCommonModel::Model::ApolloRelease::Generate->generateAll( + [map { $_->{organism} ? {%{$_->{organism}}, apollo_id => $_->{apollo_id}} : $_ } @roster], + $releaseDir, + sub { + my ($organism) = @_; + generateOrganism($organism, $releaseDir, $build, $project, $webServices, $base); + }, +); + +writeCommandFiles($result, "$releaseDir/updateCommands"); + +printf STDERR "\ngenerated %d organisms, %d failed\n", + scalar @{$results->{succeeded}}, scalar @{$results->{failed}}; + +if (@{$results->{failed}}) { + printf STDERR " %s: %s\n", $_, $results->{errors}{$_} for @{$results->{failed}}; + exit 1; +} + +exit 0; +``` + +Add these subs, and `use ApiCommonModel::Model::ApolloRelease::Generate;` at the top: + +```perl +sub generateOrganism { + my ($organism, $releaseDir, $build, $project, $webServices, $base) = @_; + + my $abbrev = $organism->{abbrev}; + my $orgDir = "$releaseDir/data/$abbrev"; + my $seqDir = "$orgDir/seq"; + make_path($seqDir); + + my $fastaDir = "$webServices/$project/build-$build/$organism->{name_for_filenames}" + . "/genomeAndProteome/fasta"; + + die "no genome fasta at $fastaDir/genome.fasta\n" unless -e "$fastaDir/genome.fasta"; + + File::Copy::cp("$fastaDir/genome.fasta", "$seqDir/$abbrev.fa") or die "cp fasta: $!"; + File::Copy::cp("$fastaDir/genome.fasta.fai", "$seqDir/$abbrev.fa.fai") or die "cp fai: $!"; + + my $twoBit = "$releaseDir/twoBit/$abbrev.2bit"; + system("faToTwoBit", "$seqDir/$abbrev.fa", $twoBit) == 0 + or die "faToTwoBit failed\n"; + + foreach my $spec (trackSpecs($abbrev, $project, $build, $webServices)) { + my ($fileName, @command) = @$spec; + my $json = runCapture(@command); + ApiCommonModel::Model::ApolloRelease::Generate->writeFile( + "$orgDir/$fileName", $json, $base); + } + + copyConf("$ENV{GUS_HOME}/lib/jbrowse/auto_generated/$abbrev/tracks.conf", + "$orgDir/tracks.conf", $base, 1); + copyConf("$ENV{GUS_HOME}/lib/jbrowse/functions.conf", + "$orgDir/functions.conf", $base, 0); + + my $refSeqs = runCapture("$ENV{GUS_HOME}/bin/jbrowseRefSeqs", + $ENV{GUS_HOME}, $project, $abbrev); + ApiCommonModel::Model::ApolloRelease::Generate->writeFile( + "$seqDir/refSeqs.json", $refSeqs, $base); + + my $trackListJson = runCapture("$ENV{GUS_HOME}/bin/jbrowseTracks", + $abbrev, $project, 0, 'geneAnnotationTracks'); + my $built = ApiCommonModel::Model::ApolloRelease::Generate->buildTrackList( + JSON::decode_json($trackListJson), $abbrev, $base); + ApiCommonModel::Model::ApolloRelease::Generate->writeFile( + "$orgDir/trackList.json", JSON::encode_json($built), $base); + + return 1; +} + +sub trackSpecs { + my ($abbrev, $project, $build, $ws) = @_; + my $bin = "$ENV{GUS_HOME}/bin"; + return ( + ['rnaseq.json', "$bin/jbrowseRnaAndChipSeqTracks", $abbrev, $project, $build, $ws, 'RNASeq', 'jbrowse'], + ['chipseq.json', "$bin/jbrowseRnaAndChipSeqTracks", $abbrev, $project, $build, $ws, 'ChIPSeq', 'jbrowse'], + ['rnaseqJunctions.json', "$bin/jbrowseRNASeqJunctionTracks", $abbrev, $project, $build, $ws, 1, 'jbrowse'], + ['organismSpecific.json', "$bin/jbrowseOrganismSpecificTracks", $abbrev, $project, 1, $build, $ws, 'jbrowse'], + ['dnaseq.json', "$bin/jbrowseDNASeqTracks", $abbrev, $project, $build, $ws, 'jbrowse'], + ); +} + +# These scripts are also served through responseFromCommand, which merges +# stderr into the response body. Anything on stderr means the output cannot be +# trusted, so refuse it here too. +sub runCapture { + my (@command) = @_; + + my $errFile = "/tmp/apolloRelease.$$.err"; + my $quoted = join(' ', map { "'$_'" } @command); + my $out = `$quoted 2>$errFile`; + my $status = $?; + + my $errSize = -s $errFile || 0; + my $errText = ''; + if ($errSize) { + open(my $fh, '<', $errFile); local $/; $errText = <$fh>; close $fh; + } + unlink $errFile; + + die "command failed (exit " . ($status >> 8) . "): $quoted\n$errText" if $status; + die "command wrote to stderr: $quoted\n$errText" if $errSize; + + return $out; +} + +sub copyConf { + my ($source, $target, $base, $stripRefseq) = @_; + + open(my $fh, '<', $source) or die "Cannot read $source: $!"; + local $/; + my $text = <$fh>; + close $fh; + + # The refseq stanza is replaced by the local IndexedFasta track. + $text =~ s/\[tracks\.refseq\][^#\[]*//s if $stripRefseq; + $text =~ s{\.\./\.\./common_tracks/}{}g; + + ApiCommonModel::Model::ApolloRelease::Generate->writeFile($target, $text, $base); +} + +sub writeCommandFiles { + my ($result, $dir) = @_; + + my $C = 'ApiCommonModel::Model::ApolloRelease::Commands'; + + open(my $curl, '>', "$dir/Apollo_curl") or die "Cannot write Apollo_curl: $!"; + print $curl $C->updateCommand($_) for @{$result->{update}}; + print $curl $C->renameCommand($_) for @{$result->{rename}}; + print $curl $C->pruneCommand($_) for @{$result->{prune_candidate}}; + close $curl; + + open(my $groovy, '>', "$dir/Apollo_groovy") or die "Cannot write Apollo_groovy: $!"; + print $groovy $C->addCommand($_) + for grep { $_->{approved} } @{$result->{add_candidate}}; + close $groovy; + + return 1; +} +``` + +Add `use JSON;` and `use File::Copy;` to the CLI's preamble. + +Prune commands are written to `Apollo_curl` **only for approved prunes** — see Step 2. + +- [ ] **Step 2: Gate prunes on approval** + +Prune candidates are proposals. Change `writeCommandFiles` to emit a prune only when the +overlay approved it: + +```perl + print $curl $C->pruneCommand($_) + for grep { $_->{approved} } @{$result->{prune_candidate}}; +``` + +and in `Reconcile::reconcile`, mark them: + +```perl + push @{$result{prune_candidate}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + common_name => $apollo->{common_name}, + annotation_count => $apollo->{annotation_count}, + approved => $overlay->{remove}{$abbrev} ? 1 : 0, + reason => $overlay->{remove}{$abbrev}, + }; +``` + +Add a test to `Model/t/reconcile.t` (bump the plan to 13 tests): + +```perl +my $overlayWithPrune = ApiCommonModel::Model::ApolloRelease::Overlay->parseString( + "remove cglaCBS138 # renamed to nglaCBS138, zero annotations\n"); +my $r3 = $R->reconcile($portal, $live, $overlayWithPrune, {}); +my ($cgla) = grep { $_->{abbrev} eq 'cglaCBS138' } @{$r3->{prune_candidate}}; +is($cgla->{approved}, 1, 'an overlay remove approves the prune'); +``` + +- [ ] **Step 3: Run the full test suite** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && cd \$PROJECT_HOME && bld ApiCommonModel/Model >/dev/null && cd ApiCommonModel && prove -v Model/t/"' +``` + +Expected: all files pass; 8 test files, 60+ assertions. + +- [ ] **Step 4: Generate three probe organisms** + +```bash +ssh cedar 'bash -lc "source /var/www/jbrestel.eupathdb.org/etc/setenv && \ + export APOLLO_API_USER=api@local.host APOLLO_API_PASS=\$(read -rsp \"apollo api pass: \" P; echo \$P) && \ + perl \$GUS_HOME/bin/createApolloReleasePackage --generate --build 71 \ + --out-dir \$HOME/apolloProbe --previous-release \$HOME/apolloConfigs/release-68/prod"' +``` + +Use `--organism tgonME49 --organism pfal3D7 --organism cdenJEC21` on the first run rather +than generating 460 organisms. + +Verify each organism dir contains all nine files from spec §6 plus `seq/`, that every JSON +parses, that every `include` names a file that exists, and that no file contains a bare +`/a/`: + +```bash +ssh cedar 'cd $HOME/apolloProbe/release-71/prod/data && for o in */; do + echo "== $o"; ls $o; grep -rl "\"/a/\|'"'"'/a/" $o && echo " BARE /a/ FOUND" || echo " urls ok" +done' +``` + +- [ ] **Step 4b: URL liveness on a sample** (spec §10) + +The absolutization assertion proves the string changed. Only a request proves it points at +something. For the three probe organisms, HEAD every distinct absolutized URL: + +```bash +ssh cedar 'python3 - < Date: Fri, 21 Aug 2026 20:33:36 -0400 Subject: [PATCH 05/42] fix(jbrowse): repair organismList history query and Postgres key casing The history query joined to an alias 'nt' absent from its FROM clause, so it threw while the script still exited 0 and printed JSON with no history. addHistoryToOrganism then read Oracle-cased keys that DBD::Pg returns lowercase. Together these meant HISTORY was always missing, and with it the annotation version every Apollo organism name depends on. Co-Authored-By: Claude Opus 5 --- Model/bin/jbrowseOrganismList | 8 +- Model/t/fixtures/portal.json | 828 ++++++++++++++++++++++++++++++++++ 2 files changed, 832 insertions(+), 4 deletions(-) create mode 100644 Model/t/fixtures/portal.json diff --git a/Model/bin/jbrowseOrganismList b/Model/bin/jbrowseOrganismList index 06c291f5f7..7ed66c26c4 100644 --- a/Model/bin/jbrowseOrganismList +++ b/Model/bin/jbrowseOrganismList @@ -33,7 +33,7 @@ my $historySql = "select h.build_number, o.public_abbrev, h.genome_source, h.gen where h.dataset_presenter_id = dd.dataset_presenter_id and dd.name like '%primary_genome_RSRC' and h.annotation_version is not null - and o.taxon_id = nt.taxon_id"; + and dd.taxon_id = o.taxon_id"; my $sh = $dbh->prepare($sql); @@ -60,11 +60,11 @@ print encode_json($result); sub addHistoryToOrganism { my ($h, $orgs) = @_; - my $publicAbbrev = $h->{PUBLIC_ABBREV}; + my $publicAbbrev = $h->{public_abbrev}; foreach(@$orgs) { - if($_->{ORGANISM_ABBREV} eq $publicAbbrev) { - push @{$_->{HISTORY}}, $h; + if($_->{organism_abbrev} eq $publicAbbrev) { + push @{$_->{history}}, $h; return; } } diff --git a/Model/t/fixtures/portal.json b/Model/t/fixtures/portal.json new file mode 100644 index 0000000000..561d338c81 --- /dev/null +++ b/Model/t/fixtures/portal.json @@ -0,0 +1,828 @@ +{ + "organisms": [ + { + "history": [ + { + "public_abbrev": "cdenJEC21", + "build_number": "51", + "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "genome_version": "GCA_000091045.1", + "annotation_source": "GenBank" + }, + { + "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "build_number": "48", + "genome_version": "Jun 16, 2016", + "public_abbrev": "cdenJEC21", + "annotation_source": "GenBank" + }, + { + "annotation_source": "GenBank", + "public_abbrev": "cdenJEC21", + "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "build_number": "44", + "genome_version": "Jun 16, 2016" + }, + { + "annotation_source": "GenBank", + "public_abbrev": "cdenJEC21", + "build_number": "43", + "annotation_version": "Jun 16, 2016", + "genome_source": "GenBank", + "genome_version": "Jun 16, 2016" + }, + { + "genome_version": "Jun 16, 2016", + "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "build_number": "41", + "public_abbrev": "cdenJEC21", + "annotation_source": "GenBank" + }, + { + "annotation_source": "GenBank", + "public_abbrev": "cdenJEC21", + "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "build_number": "37", + "genome_version": "Jun 16, 2016" + }, + { + "annotation_source": "GenBank", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016", + "genome_source": "GenBank", + "build_number": "36", + "genome_version": "Jun 16, 2016" + }, + { + "genome_version": "Jun 16, 2016", + "build_number": "29", + "annotation_version": "Jun 16, 2016", + "genome_source": "GenBank", + "public_abbrev": "cdenJEC21", + "annotation_source": "GenBank" + }, + { + "genome_source": "GenBank", + "annotation_version": "2010-08-14", + "build_number": "15", + "genome_version": "2010-08-14", + "public_abbrev": "cdenJEC21", + "annotation_source": "TIGR-Stanford" + } + ], + "is_reference_strain": "1", + "species_ncbi_tax_id": "40410", + "strain_abbrev": "JEC21", + "name_for_filenames": "CdeneoformansJEC21", + "is_annotated_genome": "1", + "name": "Cryptococcus deneoformans JEC21", + "organism_abbrev": "cdenJEC21" + }, + { + "is_reference_strain": "1", + "history": [ + { + "public_abbrev": "pfal3D7", + "build_number": "71", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "genome_version": "GCA_000002765.3", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "pfal3D7", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "build_number": "69", + "genome_version": "GCA_000002765.3", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "pfal3D7", + "genome_version": "GCA_000002765.3", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "build_number": "68", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "pfal3D7", + "build_number": "66", + "genome_source": "GenBank", + "annotation_version": "Sep 01, 2020", + "genome_version": "GCA_000002765.3", + "annotation_source": "GeneDB" + }, + { + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "build_number": "63", + "genome_version": "GCA_000002765.3", + "public_abbrev": "pfal3D7", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "genome_source": "GenBank", + "annotation_version": "Sep 01, 2020", + "build_number": "62", + "genome_version": "GCA_000002765.3", + "public_abbrev": "pfal3D7" + }, + { + "public_abbrev": "pfal3D7", + "genome_version": "GCA_000002765.3", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "build_number": "60", + "annotation_source": "GeneDB" + }, + { + "genome_version": "GCA_000002765.3", + "build_number": "57", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "public_abbrev": "pfal3D7", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "build_number": "55", + "annotation_version": "Sep 01, 2020", + "genome_source": "GenBank", + "genome_version": "GCA_000002765.3", + "public_abbrev": "pfal3D7" + }, + { + "build_number": "53", + "genome_source": "GenBank", + "annotation_version": "Sep 01, 2020", + "genome_version": "GCA_000002765.3", + "public_abbrev": "pfal3D7", + "annotation_source": "GeneDB" + }, + { + "genome_version": "GCA_000002765.3", + "genome_source": "GenBank", + "annotation_version": "Sep 01, 2020", + "build_number": "51", + "public_abbrev": "pfal3D7", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "pfal3D7", + "genome_version": "Jun 18, 2015", + "annotation_version": "Jun 18, 2015", + "genome_source": "GeneDB", + "build_number": "48" + }, + { + "annotation_source": "GeneDB", + "build_number": "43", + "annotation_version": "Jun 18, 2015", + "genome_source": "GeneDB", + "genome_version": "Jun 18, 2015", + "public_abbrev": "pfal3D7" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "pfal3D7", + "genome_version": "Jun 18, 2015", + "annotation_version": "Jun 18, 2015", + "genome_source": "GeneDB", + "build_number": "41" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "pfal3D7", + "genome_version": "Jun 18, 2015", + "build_number": "37", + "genome_source": "GeneDB", + "annotation_version": "Jun 18, 2015" + }, + { + "annotation_source": "GeneDB", + "build_number": "35", + "genome_source": "GeneDB", + "annotation_version": "Jun 18, 2015", + "genome_version": "Jun 18, 2015", + "public_abbrev": "pfal3D7" + }, + { + "annotation_source": "GeneDB", + "genome_source": "GeneDB", + "annotation_version": "Jun 18, 2015", + "build_number": "32", + "genome_version": "Jun 18, 2015", + "public_abbrev": "pfal3D7" + }, + { + "annotation_source": "GeneDB", + "genome_version": "Jun 18, 2015", + "build_number": "29", + "genome_source": "GeneDB", + "annotation_version": "Jun 18, 2015", + "public_abbrev": "pfal3D7" + }, + { + "annotation_source": "GeneDB", + "genome_source": "GeneDB", + "annotation_version": "Mar 1, 2013", + "build_number": "26", + "genome_version": "Mar 1, 2013", + "public_abbrev": "pfal3D7" + }, + { + "public_abbrev": "pfal3D7", + "build_number": "19", + "genome_source": "GeneDB", + "annotation_version": "Mar 1, 2013", + "genome_version": "Mar 1, 2013", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "pfal3D7", + "build_number": "14", + "genome_source": "GeneDB", + "annotation_version": "Feb 1, 2012", + "genome_version": "Feb 1, 2012" + }, + { + "public_abbrev": "pfal3D7", + "genome_version": "n/a", + "annotation_version": "n/a", + "genome_source": "n/a", + "build_number": "0", + "annotation_source": "n/a" + } + ], + "species_ncbi_tax_id": "5833", + "is_annotated_genome": "1", + "name_for_filenames": "Pfalciparum3D7", + "strain_abbrev": "3D7", + "organism_abbrev": "pfal3D7", + "name": "Plasmodium falciparum 3D7" + }, + { + "name_for_filenames": "NglabratusCBS138", + "strain_abbrev": "CBS138", + "is_annotated_genome": "1", + "name": "Nakaseomyces glabratus CBS 138", + "organism_abbrev": "nglaCBS138", + "history": [ + { + "annotation_source": "CGD", + "genome_version": "GCA_000002545.2", + "build_number": "69", + "annotation_version": "s02-m07-r27", + "genome_source": "GenBank", + "public_abbrev": "nglaCBS138" + }, + { + "annotation_source": "CGD", + "public_abbrev": "nglaCBS138", + "genome_source": "GenBank", + "annotation_version": "s02-m07-r27", + "build_number": "64", + "genome_version": "GCA_000002545.2" + }, + { + "annotation_source": "CGD", + "public_abbrev": "nglaCBS138", + "build_number": "53", + "annotation_version": "s02-m07-r27", + "genome_source": "GenBank", + "genome_version": "GCA_000002545.2" + }, + { + "annotation_source": "CGD", + "annotation_version": "s02-m07-r27", + "genome_source": "GenBank", + "build_number": "51", + "genome_version": "GCA_000002545.2", + "public_abbrev": "nglaCBS138" + }, + { + "annotation_source": "CGD", + "public_abbrev": "nglaCBS138", + "build_number": "39", + "genome_source": "CGD", + "annotation_version": "s02-m07-r27", + "genome_version": "s02-m07-r27" + }, + { + "annotation_version": "s02-m07-r06", + "genome_source": "CGD", + "build_number": "29", + "genome_version": "s02-m07-r06", + "public_abbrev": "nglaCBS138", + "annotation_source": "CGD" + }, + { + "annotation_source": "CGD", + "public_abbrev": "nglaCBS138", + "annotation_version": "Nov 17, 2013", + "genome_source": "CGD", + "build_number": "21", + "genome_version": "Nov 17, 2013" + } + ], + "is_reference_strain": "1", + "species_ncbi_tax_id": "5478" + }, + { + "history": [ + { + "annotation_source": "Ensembl", + "genome_source": "Ensembl", + "annotation_version": "GRCh38.p13", + "build_number": "52", + "genome_version": "GCA_000001405.28", + "public_abbrev": "hsapREF" + }, + { + "build_number": "29", + "annotation_version": "24", + "genome_source": "GENCODE", + "genome_version": "GRCh38", + "public_abbrev": "hsapREF", + "annotation_source": "GENCODE" + }, + { + "annotation_source": "GENCODE", + "public_abbrev": "hsapREF", + "annotation_version": "13", + "genome_source": "UCSC", + "build_number": "18", + "genome_version": "hg19" + } + ], + "is_reference_strain": "1", + "species_ncbi_tax_id": "9606", + "strain_abbrev": "REF", + "name_for_filenames": "HsapiensREF", + "is_annotated_genome": "1", + "name": "Homo sapiens REF", + "organism_abbrev": "hsapREF" + }, + { + "is_annotated_genome": "1", + "strain_abbrev": "ME49", + "name_for_filenames": "TgondiiME49", + "organism_abbrev": "tgonME49", + "name": "Toxoplasma gondii ME49", + "is_reference_strain": "1", + "history": [ + { + "annotation_version": "Jul 01, 2023", + "genome_source": "GenBank", + "build_number": "71", + "genome_version": "GCF_000006565.2", + "public_abbrev": "tgonME49", + "annotation_source": "Apollo" + }, + { + "annotation_source": "Apollo", + "annotation_version": "Jul 01, 2023", + "genome_source": "GenBank", + "build_number": "70", + "genome_version": "GCF_000006565.2", + "public_abbrev": "tgonME49" + }, + { + "annotation_source": "Apollo", + "public_abbrev": "tgonME49", + "genome_source": "GenBank", + "annotation_version": "Jul 01, 2023", + "build_number": "69", + "genome_version": "GCF_000006565.2" + }, + { + "genome_version": "GCF_000006565.2", + "build_number": "68", + "annotation_version": "Jul 01, 2023", + "genome_source": "GenBank", + "public_abbrev": "tgonME49", + "annotation_source": "Apollo" + }, + { + "genome_source": "GenBank", + "annotation_version": "Jul 01, 2023", + "build_number": "67", + "genome_version": "GCF_000006565.2", + "public_abbrev": "tgonME49", + "annotation_source": "Apollo" + }, + { + "public_abbrev": "tgonME49", + "genome_source": "GenBank", + "annotation_version": "Jul 01, 2023", + "build_number": "66", + "genome_version": "GCF_000006565.2", + "annotation_source": "Apollo" + }, + { + "annotation_version": "Mar 22, 2015", + "genome_source": "GenBank", + "build_number": "65", + "genome_version": "GCF_000006565.2", + "public_abbrev": "tgonME49", + "annotation_source": "GenBank" + }, + { + "genome_version": "GCF_000006565.2", + "build_number": "64", + "genome_source": "GenBank", + "annotation_version": "Mar 22, 2015", + "public_abbrev": "tgonME49", + "annotation_source": "GenBank" + }, + { + "annotation_source": "GenBank", + "genome_version": "GCF_000006565.2", + "genome_source": "GenBank", + "annotation_version": "Mar 22, 2015", + "build_number": "49", + "public_abbrev": "tgonME49" + }, + { + "genome_version": "Nov 01, 2013", + "annotation_version": "Mar 22, 2015", + "genome_source": "GenBank", + "build_number": "29", + "public_abbrev": "tgonME49", + "annotation_source": "GenBank" + }, + { + "annotation_version": "Apr 23, 2013", + "genome_source": "JCVI", + "build_number": "19", + "genome_version": "Apr 23, 2013", + "public_abbrev": "tgonME49", + "annotation_source": "JCVI" + }, + { + "public_abbrev": "tgonME49", + "genome_version": "Jul 8, 2012", + "genome_source": "JCVI", + "annotation_version": "Jul 8, 2012", + "build_number": "16", + "annotation_source": "JCVI" + }, + { + "genome_version": "Jul 23, 2008", + "genome_source": "JCVI", + "annotation_version": "Jul 23, 2008", + "build_number": "0", + "public_abbrev": "tgonME49", + "annotation_source": "JCVI" + } + ], + "species_ncbi_tax_id": "5811" + }, + { + "is_reference_strain": "1", + "history": [ + { + "annotation_source": "GenBank", + "build_number": "51", + "genome_source": "GenBank", + "annotation_version": "Mar 23, 2015", + "genome_version": "GCA_000149585.1", + "public_abbrev": "hcapNAm1" + }, + { + "annotation_source": "GenBank", + "build_number": "29", + "annotation_version": "Mar 23, 2015", + "genome_source": "GenBank", + "genome_version": "Mar 23, 2015", + "public_abbrev": "hcapNAm1" + }, + { + "annotation_version": "1", + "genome_source": "GenBank", + "build_number": "16", + "genome_version": "2009-08-10", + "public_abbrev": "hcapNAm1", + "annotation_source": "Broad" + } + ], + "species_ncbi_tax_id": "2059318", + "is_annotated_genome": "1", + "strain_abbrev": "NAm1", + "name_for_filenames": "HmississippienseNAm1", + "organism_abbrev": "hcapNAm1", + "name": "Histoplasma mississippiense (nom. inval.) NAm1" + }, + { + "is_reference_strain": "0", + "history": [ + { + "public_abbrev": "tbruLister427_2018", + "annotation_version": "Dec 17, 2018", + "genome_source": "GenBank", + "build_number": "52", + "genome_version": "GCA_900497135.1", + "annotation_source": "SiegelLab" + }, + { + "annotation_source": "SiegelLab", + "genome_version": "Dec 17, 2018", + "build_number": "42", + "annotation_version": "Dec 17, 2018", + "genome_source": "SiegelLab", + "public_abbrev": "tbruLister427_2018" + } + ], + "species_ncbi_tax_id": "5691", + "is_annotated_genome": "1", + "name_for_filenames": "TbruceiLister427_2018", + "strain_abbrev": "Lister427_2018", + "organism_abbrev": "tbruLister427_2018", + "name": "Trypanosoma brucei Lister strain 427 2018" + }, + { + "organism_abbrev": "tbruTREU927", + "name": "Trypanosoma brucei brucei TREU927", + "is_annotated_genome": "1", + "name_for_filenames": "TbruceiTREU927", + "strain_abbrev": "TREU927", + "species_ncbi_tax_id": "5691", + "is_reference_strain": "1", + "history": [ + { + "annotation_source": "VEuPathDB", + "genome_version": "GCA_000002445.1", + "genome_source": "GenBank", + "annotation_version": "Feb 23, 2024", + "build_number": "70", + "public_abbrev": "tbruTREU927" + }, + { + "genome_version": "GCA_000002445.1", + "annotation_version": "Feb 23, 2024", + "genome_source": "GenBank", + "build_number": "69", + "public_abbrev": "tbruTREU927", + "annotation_source": "VEuPathDB" + }, + { + "annotation_source": "GeneDB", + "genome_version": "GCA_000002445.1", + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "build_number": "66", + "public_abbrev": "tbruTREU927" + }, + { + "annotation_source": "GeneDB", + "genome_version": "GCA_000002445.1", + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "build_number": "65", + "public_abbrev": "tbruTREU927" + }, + { + "genome_version": "GCA_000002445.1", + "build_number": "63", + "genome_source": "GenBank", + "annotation_version": "Nov 01, 2020", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "tbruTREU927", + "build_number": "60", + "genome_source": "GenBank", + "annotation_version": "Nov 01, 2020", + "genome_version": "GCA_000002445.1", + "annotation_source": "GeneDB" + }, + { + "genome_version": "GCA_000002445.1", + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "build_number": "58", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "build_number": "57", + "genome_source": "GenBank", + "annotation_version": "Nov 01, 2020", + "genome_version": "GCA_000002445.1", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "genome_version": "GCA_000002445.1", + "build_number": "56", + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "tbruTREU927", + "genome_version": "GCA_000002445.1", + "genome_source": "GenBank", + "annotation_version": "Nov 01, 2020", + "build_number": "54", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "build_number": "52", + "genome_version": "GCA_000002445.1" + }, + { + "annotation_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "build_number": "51", + "genome_version": "Jul 10, 2018", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "genome_version": "Jul 10, 2018", + "build_number": "49", + "annotation_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "public_abbrev": "tbruTREU927" + }, + { + "public_abbrev": "tbruTREU927", + "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "build_number": "48", + "genome_version": "Jul 10, 2018", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "build_number": "46" + }, + { + "build_number": "45", + "annotation_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "genome_version": "Jul 10, 2018", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "build_number": "44", + "genome_version": "Jul 10, 2018", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "build_number": "43", + "genome_version": "Jul 10, 2018" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_version": "Jul 10, 2018", + "annotation_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "build_number": "42" + }, + { + "annotation_version": "Jul 10, 2018", + "genome_source": "GeneDB", + "build_number": "41", + "genome_version": "Jul 10, 2018", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "public_abbrev": "tbruTREU927", + "genome_version": "Jul 10, 2018", + "build_number": "39", + "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "annotation_version": "Jan 09, 2017", + "genome_source": "GeneDB", + "build_number": "38", + "genome_version": "Jan 09, 2017", + "public_abbrev": "tbruTREU927" + }, + { + "public_abbrev": "tbruTREU927", + "genome_version": "Jan 09, 2017", + "genome_source": "GeneDB", + "annotation_version": "Jan 09, 2017", + "build_number": "37", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_version": "Jan 09, 2017", + "build_number": "36", + "genome_source": "GeneDB", + "annotation_version": "Jan 09, 2017" + }, + { + "annotation_source": "GeneDB", + "genome_version": "Jan 09, 2017", + "build_number": "34", + "annotation_version": "Jan 09, 2017", + "genome_source": "GeneDB", + "public_abbrev": "tbruTREU927" + }, + { + "annotation_source": "GeneDB", + "annotation_version": "Jan 09, 2017", + "genome_source": "GeneDB", + "build_number": "33", + "genome_version": "Jan 09, 2017", + "public_abbrev": "tbruTREU927" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_source": "GeneDB", + "annotation_version": "Jan 09, 2017", + "build_number": "32", + "genome_version": "Jan 09, 2017" + }, + { + "genome_version": "Jan 09, 2017", + "annotation_version": "Jan 09, 2017", + "genome_source": "GeneDB", + "build_number": "31", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "genome_version": "May 19, 2016", + "annotation_version": "May 19, 2016", + "genome_source": "GeneDB", + "build_number": "29", + "public_abbrev": "tbruTREU927" + }, + { + "build_number": "27", + "annotation_version": "Aug 28, 2013", + "genome_source": "GeneDB", + "genome_version": "Aug 28, 2013", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "genome_version": "Aug 28, 2013", + "genome_source": "GeneDB", + "annotation_version": "Aug 28, 2013", + "build_number": "20", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + }, + { + "annotation_source": "GeneDB", + "public_abbrev": "tbruTREU927", + "genome_source": "GeneDB", + "annotation_version": "Jan 16, 2013", + "build_number": "18", + "genome_version": "Jan 16, 2013" + }, + { + "genome_version": "Oct 20, 2010", + "build_number": "0", + "genome_source": "GeneDB", + "annotation_version": "Oct 20, 2010", + "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB" + } + ] + } + ] +} \ No newline at end of file From aa8bb301ab8fd21d9a191180cb30645fe5deddc0 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:36:49 -0400 Subject: [PATCH 06/42] feat(apollo): add Portal module for the UniDB organism list Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Portal.pm | 88 ++++++++++++++++++++++++++ Model/t/portal.t | 24 +++++++ 2 files changed, 112 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Portal.pm create mode 100644 Model/t/portal.t diff --git a/Model/lib/perl/ApolloRelease/Portal.pm b/Model/lib/perl/ApolloRelease/Portal.pm new file mode 100644 index 0000000000..eaa249de97 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Portal.pm @@ -0,0 +1,88 @@ +package ApiCommonModel::Model::ApolloRelease::Portal; + +use strict; +use warnings; + +use JSON; + +# Reads the organism list produced by Model/bin/jbrowseOrganismList and +# normalises it. Keys arrive lowercase from DBD::Pg; booleans arrive as the +# strings "1"/"0". Everything downstream sees plain 0/1 and a single +# `abbrev` key. + +sub loadFromCommand { + my ($class, $projectName) = @_; + + my $gusHome = $ENV{GUS_HOME} or die "GUS_HOME is not set\n"; + my $cmd = "$gusHome/bin/jbrowseOrganismList $projectName"; + + my $json = `$cmd 2>/tmp/apolloRelease.organismList.err`; + die "jbrowseOrganismList failed (exit " . ($? >> 8) . "); see /tmp/apolloRelease.organismList.err\n" + if $?; + + my $errSize = -s "/tmp/apolloRelease.organismList.err" || 0; + die "jbrowseOrganismList wrote $errSize bytes to stderr; refusing to trust its output.\n" + . "See /tmp/apolloRelease.organismList.err\n" + if $errSize; + + return $class->_normalise(decode_json($json)); +} + +sub loadFromFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read $path: $!"; + local $/; + my $json = <$fh>; + close $fh; + + return $class->_normalise(decode_json($json)); +} + +sub _normalise { + my ($class, $decoded) = @_; + + my %byAbbrev; + + foreach my $raw (@{$decoded->{organisms}}) { + my $abbrev = $raw->{organism_abbrev}; + + $byAbbrev{$abbrev} = { + abbrev => $abbrev, + name => $raw->{name}, + name_for_filenames => $raw->{name_for_filenames}, + strain_abbrev => $raw->{strain_abbrev}, + species_taxon => $raw->{species_ncbi_tax_id}, + is_reference => $raw->{is_reference_strain} ? 1 : 0, + is_annotated => $raw->{is_annotated_genome} ? 1 : 0, + history => $raw->{history} || [], + latest_annotation_version => $class->_latestAnnotationVersion($raw->{history}), + }; + } + + return \%byAbbrev; +} + +# The annotation version belonging to the highest build number. Apollo names +# an organism " []", so this string is part of +# the organism's identity there. +sub _latestAnnotationVersion { + my ($class, $history) = @_; + + return undef unless $history && @$history; + + my $best; + foreach my $h (@$history) { + next unless defined $h->{annotation_version}; + $best = $h if !$best || $h->{build_number} > $best->{build_number}; + } + + return $best ? $best->{annotation_version} : undef; +} + +sub qualifies { + my ($class, $organism) = @_; + return ($organism->{is_reference} && $organism->{is_annotated}) ? 1 : 0; +} + +1; diff --git a/Model/t/portal.t b/Model/t/portal.t new file mode 100644 index 0000000000..316087d6f1 --- /dev/null +++ b/Model/t/portal.t @@ -0,0 +1,24 @@ +use strict; +use warnings; +use Test::More tests => 9; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Portal; + +my $P = 'ApiCommonModel::Model::ApolloRelease::Portal'; + +my $orgs = $P->loadFromFile("Model/t/fixtures/portal.json"); + +is(ref($orgs), 'HASH', 'returns a hash keyed by abbrev'); +ok(exists $orgs->{tgonME49}, 'tgonME49 present'); + +my $t = $orgs->{tgonME49}; +is($t->{abbrev}, 'tgonME49', 'abbrev normalised'); +is($t->{name_for_filenames}, 'TgondiiME49', 'name_for_filenames carried'); +is($t->{is_reference}, 1, 'is_reference is a boolean, not a string'); +is($t->{is_annotated}, 1, 'is_annotated is a boolean, not a string'); + +my $b = $orgs->{tbruLister427_2018}; +is($b->{is_reference}, 0, 'non-reference strain is 0'); + +ok(defined $t->{latest_annotation_version}, 'latest annotation version derived'); +is($P->qualifies($b), 0, 'non-reference organism does not qualify'); From e2c564f6edf8dfa4fdd608c9df3f8f961326b25d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:40:06 -0400 Subject: [PATCH 07/42] fix(apollo): coerce Portal flags and build numbers numerically The string "0.0" is true under Perl's boolean rules but zero numerically, so '? 1 : 0' on a flag would silently invert it. Coerce via looks_like_number and die on a non-numeric flag rather than guess. Compare build_number numerically by intent, skipping a non-numeric row loudly (stderr is fatal for the caller). Expose normalise() as the public seam so tests exercise the real rules without a fixture, subprocess or database. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Portal.pm | 40 +++++++++++++++++++++----- Model/t/portal.t | 19 +++++++++++- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Portal.pm b/Model/lib/perl/ApolloRelease/Portal.pm index eaa249de97..26e4aab19b 100644 --- a/Model/lib/perl/ApolloRelease/Portal.pm +++ b/Model/lib/perl/ApolloRelease/Portal.pm @@ -4,6 +4,7 @@ use strict; use warnings; use JSON; +use Scalar::Util qw(looks_like_number); # Reads the organism list produced by Model/bin/jbrowseOrganismList and # normalises it. Keys arrive lowercase from DBD::Pg; booleans arrive as the @@ -25,7 +26,7 @@ sub loadFromCommand { . "See /tmp/apolloRelease.organismList.err\n" if $errSize; - return $class->_normalise(decode_json($json)); + return $class->normalise(decode_json($json)); } sub loadFromFile { @@ -36,10 +37,13 @@ sub loadFromFile { my $json = <$fh>; close $fh; - return $class->_normalise(decode_json($json)); + return $class->normalise(decode_json($json)); } -sub _normalise { +# Public because it is the seam: it takes an already-decoded document, so the +# normalisation rules can be exercised without a fixture file, a subprocess or +# a database. Both loaders are thin wrappers around it. +sub normalise { my ($class, $decoded) = @_; my %byAbbrev; @@ -53,8 +57,8 @@ sub _normalise { name_for_filenames => $raw->{name_for_filenames}, strain_abbrev => $raw->{strain_abbrev}, species_taxon => $raw->{species_ncbi_tax_id}, - is_reference => $raw->{is_reference_strain} ? 1 : 0, - is_annotated => $raw->{is_annotated_genome} ? 1 : 0, + is_reference => _boolean($raw->{is_reference_strain}), + is_annotated => _boolean($raw->{is_annotated_genome}), history => $raw->{history} || [], latest_annotation_version => $class->_latestAnnotationVersion($raw->{history}), }; @@ -63,9 +67,24 @@ sub _normalise { return \%byAbbrev; } +# is_reference_strain / is_annotated_genome arrive as the strings "1"/"0". +# Coerce numerically: the string "0.0" is TRUE under Perl's boolean rules but +# 0 numerically, and treating it as true would silently invert the flag. +# These are numeric flags in the database, so a non-numeric value means the +# upstream query changed shape -- fail rather than guess. +sub _boolean { + my ($value) = @_; + + return 0 unless defined $value; + die "expected a numeric flag, got '$value'\n" unless looks_like_number($value); + + return $value + 0 ? 1 : 0; +} + # The annotation version belonging to the highest build number. Apollo names # an organism " []", so this string is part of -# the organism's identity there. +# the organism's identity there. build_number is a string and is not always +# an integer ("19.9" occurs), so compare numerically by intent. sub _latestAnnotationVersion { my ($class, $history) = @_; @@ -74,7 +93,14 @@ sub _latestAnnotationVersion { my $best; foreach my $h (@$history) { next unless defined $h->{annotation_version}; - $best = $h if !$best || $h->{build_number} > $best->{build_number}; + + unless (looks_like_number($h->{build_number})) { + my $shown = defined $h->{build_number} ? $h->{build_number} : '(undef)'; + warn "skipping history row with non-numeric build_number '$shown'\n"; + next; + } + + $best = $h if !$best || ($h->{build_number} + 0) > ($best->{build_number} + 0); } return $best ? $best->{annotation_version} : undef; diff --git a/Model/t/portal.t b/Model/t/portal.t index 316087d6f1..8aaa969787 100644 --- a/Model/t/portal.t +++ b/Model/t/portal.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 9; +use Test::More tests => 12; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; @@ -22,3 +22,20 @@ is($b->{is_reference}, 0, 'non-reference strain is 0'); ok(defined $t->{latest_annotation_version}, 'latest annotation version derived'); is($P->qualifies($b), 0, 'non-reference organism does not qualify'); + +# "0.0" is truthy as a Perl string but zero numerically. Pin the coercion so a +# future upstream change to the flag's format cannot silently invert it. +my $synthetic = $P->normalise({organisms => [ + {organism_abbrev => 'zeroPointZero', is_reference_strain => '0.0', is_annotated_genome => '1', + name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', + history => [{build_number => '19.9', annotation_version => 'older'}, + {build_number => '71', annotation_version => 'newest'}]}, +]}); +is($synthetic->{zeroPointZero}{is_reference}, 0, '"0.0" normalises to false, not true'); +is($synthetic->{zeroPointZero}{latest_annotation_version}, 'newest', + 'highest build number wins with a decimal build in the set'); + +eval { $P->normalise({organisms => [ + {organism_abbrev => 'bogus', is_reference_strain => 'yes', is_annotated_genome => '1', history => []}, +]}) }; +like($@, qr/expected a numeric flag/, 'a non-numeric flag is a hard error, not a guess'); From 6424d5698f11a31db3bb275b98e05c6ffe7e4c3d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:42:21 -0400 Subject: [PATCH 08/42] test(apollo): make the build_number ordering assertion discriminate The synthetic history used 19.9 and 71, but "71" gt "19.9" lexically too, so the old string comparison would have passed it. Add a build_number of 9, which is lexically larger than both and numerically smaller: lexical ordering now picks 'older' and only numeric ordering yields 'newest'. Co-Authored-By: Claude Opus 5 --- Model/t/portal.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Model/t/portal.t b/Model/t/portal.t index 8aaa969787..96a2ca2e46 100644 --- a/Model/t/portal.t +++ b/Model/t/portal.t @@ -28,12 +28,13 @@ is($P->qualifies($b), 0, 'non-reference organism does not qualify'); my $synthetic = $P->normalise({organisms => [ {organism_abbrev => 'zeroPointZero', is_reference_strain => '0.0', is_annotated_genome => '1', name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', - history => [{build_number => '19.9', annotation_version => 'older'}, + history => [{build_number => '9', annotation_version => 'older'}, + {build_number => '19.9', annotation_version => 'middle'}, {build_number => '71', annotation_version => 'newest'}]}, ]}); is($synthetic->{zeroPointZero}{is_reference}, 0, '"0.0" normalises to false, not true'); is($synthetic->{zeroPointZero}{latest_annotation_version}, 'newest', - 'highest build number wins with a decimal build in the set'); + 'highest build number wins numerically: beats lexically-larger "9" and a decimal'); eval { $P->normalise({organisms => [ {organism_abbrev => 'bogus', is_reference_strain => 'yes', is_annotated_genome => '1', history => []}, From 3ec2195bcd64b4db15e1a7467623f2045e0f2202 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:46:13 -0400 Subject: [PATCH 09/42] feat(apollo): add Apollo module for the live prod roster Organisms are keyed by the abbrev parsed from Apollo's directory field, never by commonName, which curators can edit in the GUI. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Apollo.pm | 112 +++++++++++++++++++++++++ Model/t/apollo.t | 75 +++++++++++++++++ Model/t/fixtures/apollo.json | 22 +++++ 3 files changed, 209 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Apollo.pm create mode 100644 Model/t/apollo.t create mode 100644 Model/t/fixtures/apollo.json diff --git a/Model/lib/perl/ApolloRelease/Apollo.pm b/Model/lib/perl/ApolloRelease/Apollo.pm new file mode 100644 index 0000000000..80b16d55d3 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Apollo.pm @@ -0,0 +1,112 @@ +package ApiCommonModel::Model::ApolloRelease::Apollo; + +use strict; +use warnings; + +use JSON; +use LWP::UserAgent; +use HTTP::Request::Common qw(POST); + +# Reads the live prod Apollo organism roster. This is the SEED for the +# release roster -- it is the only record of what curators actually decided. +# +# Organisms are keyed by the abbrev parsed out of `directory`, never by +# commonName: `directory` is machine-written by our own update commands, +# while commonName is editable in the Apollo GUI. + +sub loadFromApi { + my ($class) = @_; + + my $url = $ENV{APOLLO_API_URL} || 'https://apollo-api.veupathdb.org'; + my $user = $ENV{APOLLO_API_USER} or die "APOLLO_API_USER is not set\n"; + my $pass = $ENV{APOLLO_API_PASS} or die "APOLLO_API_PASS is not set\n"; + + my $agent = LWP::UserAgent->new(timeout => 900); + my $response = $agent->request( + POST "$url/organism/findAllOrganisms", + Content_Type => 'form-data', + Content => [username => $user, password => $pass], + ); + + die "Apollo API request failed: " . $response->status_line . "\n" + unless $response->is_success; + + my $decoded = decode_json($response->content); + + # An empty roster would make every organism look new, and the generated + # commands would try to re-add the entire set. Fail instead. + die "Apollo returned no organisms from $url.\n" + . "Check APOLLO_API_USER/PASS, and check that you are running this on a\n" + . "Penn host -- the API is IP-restricted.\n" + unless @$decoded; + + return $class->normalise($decoded); +} + +sub loadFromFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read $path: $!"; + local $/; + my $json = <$fh>; + close $fh; + + return $class->normalise(decode_json($json)); +} + +# Public because it is the seam: it takes an already-decoded document, so the +# normalisation rules can be exercised without a fixture file, an HTTP call or +# API credentials. Both loaders are thin wrappers around it. +sub normalise { + my ($class, $decoded) = @_; + + my %byAbbrev; + + foreach my $raw (@$decoded) { + my $directory = $raw->{directory} || ''; + $directory =~ s{/+$}{}; + + my ($abbrev) = $directory =~ m{([^/]+)$}; + + unless ($abbrev) { + warn "Apollo organism id $raw->{id} has an unparseable directory '$raw->{directory}'; skipping\n"; + next; + } + + # Two Apollo organisms sharing a directory is corruption, not a shape we + # can normalise. Overwriting silently would drop one of them from the + # roster, and the diff against the portal would then generate commands + # for whichever one happened to be last -- including deleting curated + # annotations that belong to the other. Refuse to guess. + if (exists $byAbbrev{$abbrev}) { + die "Apollo has two organisms with directory '$raw->{directory}' " + . "(ids $byAbbrev{$abbrev}{id} and $raw->{id}).\n" + . "Resolve this in Apollo before generating release commands.\n"; + } + + $byAbbrev{$abbrev} = { + abbrev => $abbrev, + id => $raw->{id}, + common_name => $raw->{commonName}, + directory => $raw->{directory}, + blatdb => $raw->{blatdb}, + annotation_count => $raw->{annotationCount} || 0, + public_mode => $raw->{publicMode} ? 1 : 0, + }; + } + + return \%byAbbrev; +} + +# Apollo names an organism " []". Compare only +# the part before the bracket. +sub commonNameDisagrees { + my ($class, $apolloOrganism, $portalName) = @_; + + my $live = $apolloOrganism->{common_name} || ''; + $live =~ s{\s*\[[^\]]*\]\s*$}{}; + + return ($live eq $portalName) ? 0 : 1; +} + +1; diff --git a/Model/t/apollo.t b/Model/t/apollo.t new file mode 100644 index 0000000000..42fdb3322f --- /dev/null +++ b/Model/t/apollo.t @@ -0,0 +1,75 @@ +use strict; +use warnings; +use Test::More tests => 15; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Apollo; + +my $A = 'ApiCommonModel::Model::ApolloRelease::Apollo'; + +my $live = $A->loadFromFile("Model/t/fixtures/apollo.json"); + +is(ref($live), 'HASH', 'returns a hash keyed by abbrev'); +is(scalar(keys %$live), 5, 'all five organisms parsed'); + +my $t = $live->{tgonME49}; +is($t->{id}, 1484940, 'numeric id carried'); +is($t->{annotation_count}, 10, 'annotation count carried'); +is($t->{abbrev}, 'tgonME49', 'abbrev parsed from directory'); + +my $c = $live->{cneoJEC21}; +is($c->{annotation_count}, 14, 'the organism we must not lose is parsed'); + +# commonName is curator-editable, so it is cross-checked, never matched on. +is($A->commonNameDisagrees($live->{pfal3D7}, 'Plasmodium falciparum 3D7'), 1, + 'GUI-edited common name is flagged'); +is($A->commonNameDisagrees($live->{tgonME49}, 'Toxoplasma gondii ME49'), 0, + 'matching common name is not flagged'); + +# Not every Apollo organism carries an annotation-version bracket. The strip +# is anchored, so a bracket-free name must compare whole rather than being +# treated as a mismatch (which would raise a cross-check warning on every +# such organism, every release). +is($A->commonNameDisagrees({common_name => 'Plasmodium falciparum 3D7'}, + 'Plasmodium falciparum 3D7'), 0, + 'bracket-free common name still matches'); + +# publicMode arrives as a JSON boolean object, which is an object reference in +# Perl. Pin the coercion so downstream comparisons see plain 0/1. +is($live->{tgonME49}{public_mode}, 1, 'publicMode true becomes 1'); +is($live->{cglaCBS138}{public_mode}, 0, 'publicMode false becomes 0'); + +# annotationCount is the value that decides whether an organism can be safely +# removed. A missing key must read as 0, not undef -- undef would warn under +# numeric comparison and could compare as "no annotations" by accident anyway. +my $noCount = $A->normalise([ + {id => 1, commonName => 'X', directory => '/data/apollo_data/xxxx'}, +]); +is($noCount->{xxxx}{annotation_count}, 0, 'absent annotationCount defaults to 0'); + +# A trailing slash on directory is the same organism, not a different one. +my $slashed = $A->normalise([ + {id => 2, commonName => 'Y', directory => '/data/apollo_data/tgonME49/'}, +]); +is_deeply([keys %$slashed], ['tgonME49'], 'trailing slash on directory is stripped'); + +# Two Apollo organisms sharing a directory is corruption: silently keeping the +# last one would hide an organism from the diff and could generate a delete +# against the wrong record. Fail loudly instead. +eval { + $A->normalise([ + {id => 10, commonName => 'A', directory => '/data/apollo_data/dupe', annotationCount => 0}, + {id => 11, commonName => 'B', directory => '/data/apollo_data/dupe', annotationCount => 7}, + ]); +}; +like($@, qr/two organisms with directory .*dupe.*\n?.*ids 10 and 11/s, + 'duplicate directory is a fatal error naming both ids'); + +# A directory that yields no final path segment must be skipped, not keyed +# under the empty string -- which would collide across every such record and +# put a nameless entry into the roster the release commands iterate. +my @warnings; +my $bad = do { + local $SIG{__WARN__} = sub { push @warnings, $_[0] }; + $A->normalise([{id => 99, commonName => 'Z', directory => ''}]); +}; +is_deeply($bad, {}, 'organism with an unparseable directory is skipped, not keyed as ""'); diff --git a/Model/t/fixtures/apollo.json b/Model/t/fixtures/apollo.json new file mode 100644 index 0000000000..9cf699cbd6 --- /dev/null +++ b/Model/t/fixtures/apollo.json @@ -0,0 +1,22 @@ +[ + {"id": 1484940, "commonName": "Toxoplasma gondii ME49 [Jul 01, 2023]", + "directory": "/data/apollo_data/tgonME49", + "blatdb": "/data/apollo_data/twoBit/tgonME49.2bit", + "annotationCount": 10, "publicMode": true}, + {"id": 2452162, "commonName": "Cryptococcus neoformans var. neoformans JEC21 [Jun 16, 2016]", + "directory": "/data/apollo_data/cneoJEC21", + "blatdb": "/data/apollo_data/twoBit/cneoJEC21.2bit", + "annotationCount": 14, "publicMode": true}, + {"id": 5146948, "commonName": "Candida glabrata CBS 138 [s02-m07-r27]", + "directory": "/data/apollo_data/cglaCBS138", + "blatdb": "/data/apollo_data/twoBit/cglaCBS138.2bit", + "annotationCount": 0, "publicMode": false}, + {"id": 9000001, "commonName": "Trypanosoma brucei Lister strain 427 2018 [May 01, 2019]", + "directory": "/data/apollo_data/tbruLister427_2018", + "blatdb": "/data/apollo_data/twoBit/tbruLister427_2018.2bit", + "annotationCount": 3, "publicMode": true}, + {"id": 9000002, "commonName": "Something Renamed In The Gui", + "directory": "/data/apollo_data/pfal3D7", + "blatdb": "/data/apollo_data/twoBit/pfal3D7.2bit", + "annotationCount": 3, "publicMode": true} +] From 00acbf92a84de527a097336c4608344fefd55edf Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:49:24 -0400 Subject: [PATCH 10/42] Apollo: an update must never change an organism's visibility Measured against prod: 17 organisms have publicMode=false, and 16 of them are on the portal with 15 qualifying as reference+annotated. They are curator-hidden, not retired -- the workflow Paul's notes describe for hiding an organism from users while admins retain access. Three carry annotations (iscaPalLabHiFi 20, treeQM6a 11, etenHoughton2021 2). Paul's updateOrganismInfo curl hardcodes "publicMode":"true", so running the generated commands re-publishes all 17. The plan reproduced that bug faithfully. Update and rename now echo back the visibility Apollo currently holds, and refuse to run when it is unknown rather than defaulting. Only an approved prune may change visibility. Co-Authored-By: Claude Opus 5 --- .../2026-08-21-apollo-release-package.md | 38 +++++++++++++++---- ...026-08-21-apollo-release-package-design.md | 8 +++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/plans/2026-08-21-apollo-release-package.md b/docs/superpowers/plans/2026-08-21-apollo-release-package.md index 3bbeb685c3..f90a8a8ff8 100644 --- a/docs/superpowers/plans/2026-08-21-apollo-release-package.md +++ b/docs/superpowers/plans/2026-08-21-apollo-release-package.md @@ -868,6 +868,7 @@ sub reconcile { apollo_id => $apollo->{id}, annotation_count => $apollo->{annotation_count}, organism => $target, + public_mode => $apollo->{public_mode}, }; next; } @@ -899,9 +900,12 @@ sub reconcile { } push @{$result{update}}, { - abbrev => $abbrev, - apollo_id => $apollo->{id}, - organism => $organism, + abbrev => $abbrev, + apollo_id => $apollo->{id}, + organism => $organism, + # Echoed straight back in the update command. 17 organisms are hidden + # by curators; forcing publicMode=true would silently re-publish them. + public_mode => $apollo->{public_mode}, }; } @@ -1641,14 +1645,14 @@ Create `Model/t/commands.t`: ```perl use strict; use warnings; -use Test::More tests => 8; +use Test::More tests => 10; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Commands; my $C = 'ApiCommonModel::Model::ApolloRelease::Commands'; my $update = $C->updateCommand({ - abbrev => 'tgonME49', apollo_id => 1484940, + abbrev => 'tgonME49', apollo_id => 1484940, public_mode => 1, }); like($update, qr/"id":"1484940"/, 'update targets the numeric id'); like($update, qr{/data/apollo_data/tgonME49}, 'points at the organism directory'); @@ -1656,7 +1660,7 @@ like($update, qr{/data/apollo_data/twoBit/tgonME49\.2bit}, 'and at its blatdb'); unlike($update, qr/GFERsVNiX5BQ|password":"[^\$]/, 'no literal password in output'); my $rename = $C->renameCommand({ - from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', apollo_id => 2452162, + from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', apollo_id => 2452162, public_mode => 1, organism => {name => 'Cryptococcus deneoformans JEC21', latest_annotation_version => 'Jun 16, 2016'}, }); like($rename, qr/2452162/, 'rename targets the EXISTING apollo id'); @@ -1665,6 +1669,12 @@ like($rename, qr/Cryptococcus deneoformans JEC21 \[Jun 16, 2016\]/, 'renamed wit my $prune = $C->pruneCommand({abbrev => 'cglaCBS138', apollo_id => 5146948}); like($prune, qr/"publicMode":"false"/, 'prune unpublishes rather than deleting'); + +# 17 live organisms are curator-hidden; an update must never re-publish them. +my $hidden = $C->updateCommand({abbrev => 'treeQM6a', apollo_id => 9999, public_mode => 0}); +like($hidden, qr/"publicMode":"false"/, 'a hidden organism stays hidden through an update'); +eval { $C->updateCommand({abbrev => 'x', apollo_id => 1}) }; +like($@, qr/refusing to guess visibility/, 'missing public_mode is an error, not a default'); ``` - [ ] **Step 2: Run it and watch it fail** @@ -1715,10 +1725,18 @@ sub _updateOrganismInfo { return qq{curl -X POST -H "Content-Type: application/json" --data '{$json}' $API/organism/updateOrganismInfo\n}; } +# An update must NOT change visibility. Publishing and unpublishing are +# curation decisions; 17 live organisms are deliberately hidden, 3 of them with +# annotations. The previous script hardcoded "publicMode":"true" here, which +# re-published every one of them. Echo back what Apollo currently holds. sub updateCommand { my ($class, $entry) = @_; + + die "updateCommand: public_mode missing for $entry->{abbrev}; refusing to guess visibility\n" + unless defined $entry->{public_mode}; + return $class->_updateOrganismInfo($entry->{apollo_id}, $entry->{abbrev}, - {publicMode => 'true'}); + {publicMode => $entry->{public_mode} ? 'true' : 'false'}); } # A rename repoints the EXISTING organism, preserving its annotations. The @@ -1733,8 +1751,12 @@ sub renameCommand { ? "$organism->{name} [$version]" : $organism->{name}; + die "renameCommand: public_mode missing for $entry->{from_abbrev}; refusing to guess visibility\n" + unless defined $entry->{public_mode}; + return $class->_updateOrganismInfo($entry->{apollo_id}, $entry->{to_abbrev}, - {publicMode => 'true', commonName => $name}); + {publicMode => $entry->{public_mode} ? 'true' : 'false', + commonName => $name}); } # Prune unpublishes. It is reversible, and Apollo's API has no delete in use. diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 596b68bd14..5d59e49d8c 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -43,6 +43,7 @@ Every row measured on 2026-08-21, not inferred. | 12 | Species taxon ID cannot detect a rename | `cneoJEC21` was species 5207; `cdenJEC21` is 40410. `cglaCBS138` (5478 → *Nakaseomyces*) likewise. | | 13 | The renamed genome is byte-identical | b68 `cneoJEC21.fa.fai` vs b71 `CdeneoformansJEC21/genome.fasta.fai`: 14 sequences, identical names and lengths. | | 14 | `applicationType=apollo` is unimplemented | `Store::makeUrlTemplate` is `die "TODO: make apollo work"`. `getConfigurationObject` dispatches `apollo` to the abstract `getApolloObject`; **39** classes implement `getJBrowseObject`, **1** implements `getApolloObject`. | +| 15 | 17 live organisms are deliberately hidden, and the update command would un-hide them | `publicMode=false` on 17 organisms; 16 are on the portal and 15 qualify as reference+annotated, so they are curator-hidden, not retired. 3 carry annotations (`iscaPalLabHiFi` 20, `treeQM6a` 11, `etenHoughton2021` 2). Paul's `updateOrganismInfo` curl hardcodes `"publicMode":"true"`, so running the generated commands re-publishes all 17. | ### Set sizes as of build 71 @@ -273,7 +274,12 @@ the Oracle-cased keys in `addHistoryToOrganism`. 3. Add and prune require an overlay entry; neither is inferred from the criteria alone. 4. A rename and an add for the same genome cannot both be emitted. 5. Every generated file passes the no-bare-`/a/` check before the run is declared successful. -6. Credentials come from the environment. The repo contains no password. +6. **An update never changes an organism's `publicMode`.** It is echoed back from the value + Apollo currently holds. 17 organisms are deliberately hidden by curators (fact 15); a + hardcoded `"publicMode":"true"` — which is what the previous script emitted — silently + re-publishes all of them, three with annotations. Publishing and unpublishing are curation + decisions, and the only bucket permitted to change that state is an approved prune. +7. Credentials come from the environment. The repo contains no password. ## 10. Verification From 0eeeabc379694bee3fc0e554acd8f01aa6e14638 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:49:49 -0400 Subject: [PATCH 11/42] test(apollo): pin the non-numeric build_number skip path Covers the third hardening behaviour, which had no test. Verified by mutation: deleting the looks_like_number guard makes test 13 fail. The obvious form of this test does not discriminate. A bad row among good rows passes with the guard deleted too, because 'not-a-number' + 0 is 0 and any real build number beats it -- row order does not change that. So the case is pinned with a second organism whose only history row is the bad one: with the guard it yields no latest version, without it the row becomes $best and returns 'bogus'. Co-Authored-By: Claude Opus 5 --- Model/t/portal.t | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Model/t/portal.t b/Model/t/portal.t index 96a2ca2e46..8533b49e58 100644 --- a/Model/t/portal.t +++ b/Model/t/portal.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 12; +use Test::More tests => 13; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; @@ -40,3 +40,27 @@ eval { $P->normalise({organisms => [ {organism_abbrev => 'bogus', is_reference_strain => 'yes', is_annotated_genome => '1', history => []}, ]}) }; like($@, qr/expected a numeric flag/, 'a non-numeric flag is a hard error, not a guess'); + +# A non-numeric build_number is skipped deliberately, with our own warning -- +# not left to Perl's numeric comparison, whose "isn't numeric" warning would +# go to stderr, which loadFromCommand treats as fatal. +# +# Both organisms are needed to pin the skip. messyBuilds shows a bad row does +# not corrupt the ordering of the good rows around it -- but that alone would +# pass even with the guard deleted, because "not-a-number" + 0 is 0 and any +# real build number beats it. onlyBad is the discriminating case: with the +# guard the row is skipped and there is no latest version; without it the row +# becomes $best by default and 'bogus' is returned as the annotation version. +my $messy = $P->normalise({organisms => [ + {organism_abbrev => 'messyBuilds', is_reference_strain => '1', is_annotated_genome => '1', + name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', + history => [{build_number => 'not-a-number', annotation_version => 'bogus'}, + {build_number => '9', annotation_version => 'older'}, + {build_number => '71', annotation_version => 'newest'}]}, + {organism_abbrev => 'onlyBad', is_reference_strain => '1', is_annotated_genome => '1', + name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', + history => [{build_number => 'not-a-number', annotation_version => 'bogus'}]}, +]}); +is(join('|', map { $messy->{$_}{latest_annotation_version} // 'undef' } qw(messyBuilds onlyBad)), + 'newest|undef', + 'a non-numeric build_number is skipped: it neither wins nor disturbs the good rows'); From d2229d494e4d99e0ceefe5e8b2160d9d91173169 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:54:32 -0400 Subject: [PATCH 12/42] feat(apollo): add roster overlay parser and seeded overlay file Every entry requires a reason. The file it replaces was a hardcoded __DATA__ block that froze two builds ago, where decisions and staleness had become indistinguishable. Co-Authored-By: Claude Opus 5 --- Model/data/apollo/roster-overlay.txt | 38 +++++++++++++ Model/lib/perl/ApolloRelease/Overlay.pm | 68 +++++++++++++++++++++++ Model/t/overlay.t | 71 +++++++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 Model/data/apollo/roster-overlay.txt create mode 100644 Model/lib/perl/ApolloRelease/Overlay.pm create mode 100644 Model/t/overlay.t diff --git a/Model/data/apollo/roster-overlay.txt b/Model/data/apollo/roster-overlay.txt new file mode 100644 index 0000000000..3f72288cd9 --- /dev/null +++ b/Model/data/apollo/roster-overlay.txt @@ -0,0 +1,38 @@ +# Roster overlay for the Apollo release package. +# +# The roster is seeded from LIVE PROD APOLLO. This file records the human +# decisions layered on top of it. Every line needs a reason: this file is the +# only written record of why an organism is or is not in Apollo. +# +# add # who approved, when, why +# remove # who approved, when, why + +# --- Host genomes ------------------------------------------------------- +# Apollo curates pathogens. The previous script excluded these implicitly, +# via an @databases array that omitted HostDB and SchistoDB. On the UniDB +# portal that filter does not exist, so the exclusion has to be stated. + +remove hsapREF # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmusC57BL6J # host genome (seeded 2026-08-21 from b68 behaviour) +remove rnorBNNHsdMcwi # host genome (seeded 2026-08-21 from b68 behaviour) +remove btauHereford # host genome (seeded 2026-08-21 from b68 behaviour) +remove clupfamiliarisSID07034 # host genome (seeded 2026-08-21 from b68 behaviour) +remove ggalbGalGal1 # host genome (seeded 2026-08-21 from b68 behaviour) +remove cpor2N # host genome (seeded 2026-08-21 from b68 behaviour) +remove mfasREF # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmulAG07107 # host genome (seeded 2026-08-21 from b68 behaviour) +remove mmyomMyoMyo1 # host genome (seeded 2026-08-21 from b68 behaviour) +remove dmeliso-1 # host genome (seeded 2026-08-21 from b68 behaviour) + +# --- Model fungi -------------------------------------------------------- +# In FungiDB, qualify on the criteria, never been in Apollo. Seeded to match +# existing behaviour, but nobody has consciously decided this -- raise with the +# curation team. + +remove scerS288C # model organism, never in Apollo; decision unconfirmed +remove spom972h # model organism, never in Apollo; decision unconfirmed + +# --- Deliberately NOT listed here --------------------------------------- +# hcapNAm1 (Histoplasma mississippiense NAm1) is a FungiDB pathogen that +# qualifies and is absent from Apollo. It is left in the add-candidate bucket +# for the curation team rather than suppressed here. diff --git a/Model/lib/perl/ApolloRelease/Overlay.pm b/Model/lib/perl/ApolloRelease/Overlay.pm new file mode 100644 index 0000000000..47d466ce59 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Overlay.pm @@ -0,0 +1,68 @@ +package ApiCommonModel::Model::ApolloRelease::Overlay; + +use strict; +use warnings; + +# The human decisions layered over the live Apollo roster: +# +# add # who approved it, when, why +# remove # who approved it, when, why +# +# The reason is mandatory. This file exists to record provenance; a line +# without it is indistinguishable from the hardcoded __DATA__ block this +# replaces, which froze two builds ago and drifted silently. + +sub parseFile { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or die "Cannot read roster overlay $path: $!"; + local $/; + my $text = <$fh>; + close $fh; + + return $class->parseString($text); +} + +sub parseString { + my ($class, $text) = @_; + + my %overlay = (add => {}, remove => {}); + my $lineNumber = 0; + + foreach my $line (split /\n/, $text) { + $lineNumber++; + + next if $line =~ /^\s*$/; + next if $line =~ /^\s*#/; + + my ($directive, $abbrev, $reason) = + $line =~ /^\s*(\S+)\s+(\S+)\s*#\s*(\S.*?)\s*$/; + + unless (defined $directive) { + my ($lone) = $line =~ /^\s*(\S+)\s*$/; + die "roster overlay line $lineNumber: missing organism or reason: '$line'\n" + if $lone; + die "roster overlay line $lineNumber: every entry needs a non-empty '# reason': '$line'\n"; + } + + die "roster overlay line $lineNumber: unknown directive '$directive' (expected add or remove)\n" + unless $directive eq 'add' || $directive eq 'remove'; + + my $other = $directive eq 'add' ? 'remove' : 'add'; + die "roster overlay line $lineNumber: $abbrev is listed as both add and remove\n" + if exists $overlay{$other}{$abbrev}; + + # A repeat of the same directive would silently discard the first line's + # reason. This file's whole purpose is provenance, so losing one is a + # defect, not a merge: refuse it and make the author reconcile the two. + die "roster overlay line $lineNumber: $abbrev is listed as '$directive' more than once; " + . "merge the reasons into one line\n" + if exists $overlay{$directive}{$abbrev}; + + $overlay{$directive}{$abbrev} = $reason; + } + + return \%overlay; +} + +1; diff --git a/Model/t/overlay.t b/Model/t/overlay.t new file mode 100644 index 0000000000..9f352bc3c4 --- /dev/null +++ b/Model/t/overlay.t @@ -0,0 +1,71 @@ +use strict; +use warnings; +use Test::More tests => 12; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Overlay; + +my $O = 'ApiCommonModel::Model::ApolloRelease::Overlay'; + +my $text = <<'OVERLAY'; +# comment line, ignored +remove hsapREF # host genome; Apollo curates pathogens + +add someNewOrg # approved by Uli 2026-08-21 +OVERLAY + +my $overlay = $O->parseString($text); + +is_deeply([sort keys %{$overlay->{remove}}], ['hsapREF'], 'remove parsed'); +is_deeply([sort keys %{$overlay->{add}}], ['someNewOrg'], 'add parsed'); +like($overlay->{remove}{hsapREF}, qr/host genome/, 'reason retained'); +like($overlay->{add}{someNewOrg}, qr/Uli/, 'approver retained'); + +eval { $O->parseString("bogus hsapREF # what\n") }; +like($@, qr/unknown directive 'bogus'/, 'unknown directive rejected'); + +eval { $O->parseString("add hsapREF # x\nremove hsapREF # y\n") }; +like($@, qr/both add and remove/, 'contradictory entry rejected'); + +eval { $O->parseString("add\n") }; +like($@, qr/missing organism/, 'malformed line rejected'); + +# --- gaps considered in review ----------------------------------------- + +# BEHAVIOUR CHANGE (deliberate): the same organism repeated with the SAME +# directive used to let the second line silently overwrite the first, throwing +# away one of the two reasons. This file's only job is to record provenance, +# so silently discarding a reason is exactly the failure mode it exists to +# prevent -- and two lines for one organism means two people wrote down two +# different justifications, which a human must reconcile. Now a hard error. +eval { $O->parseString("remove hsapREF # host genome\nremove hsapREF # not loaded here\n") }; +like($@, qr/more than once/, 'duplicate same-directive entry rejected, not silently merged'); + +# BEHAVIOUR CHANGE (deliberate): the reason group was `(.+?)` with `\s*` on +# both sides, which happily matched an empty string -- "add x # " parsed and +# stored an empty reason. A blank reason satisfies the syntax while defeating +# the point, so the group now requires a leading non-space character. +eval { $O->parseString("add someOrg # \n") }; +like($@, qr/needs a non-empty '# reason'/, 'whitespace-only reason rejected'); + +# A '#' is ordinary text once the reason has started -- ticket refs and +# "issue #123" style provenance are exactly what people will write, and +# truncating the reason at the second '#' would quietly drop the useful half. +my $hashy = $O->parseString("add someOrg # approved by Uli, see ticket #4412 # dup marker\n"); +is($hashy->{add}{someOrg}, 'approved by Uli, see ticket #4412 # dup marker', + 'a # inside the reason is kept verbatim, not treated as a second comment'); + +# The overlay path is passed in by a caller, so a typo or a missed `bld` is the +# likely failure. Pin that the error names the path and the file's role rather +# than dying somewhere later on an undef. +eval { $O->parseFile("/nonexistent/apollo/roster-overlay.txt") }; +like($@, qr{Cannot read roster overlay /nonexistent/apollo/roster-overlay\.txt}, + 'a missing overlay file gives an actionable error naming the path'); + +# Parsing the real shipped file is the only test that can catch a seeded line +# that stopped being parseable; it also pins that hcapNAm1 stays an add +# candidate rather than being quietly suppressed. +my $shipped = $O->parseFile($ENV{GUS_HOME} . "/data/ApiCommonModel/Model/apollo/roster-overlay.txt"); +is(join('/', scalar(keys %{$shipped->{remove}}), scalar(keys %{$shipped->{add}}), + ($shipped->{remove}{hcapNAm1} ? 'suppressed' : 'candidate')), + '13/0/candidate', + 'the shipped overlay parses: 13 removes, 0 adds, hcapNAm1 left as an add candidate'); From 4df20cd9a511d46e0b004d395ae03d01914ae2fd Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:54:43 -0400 Subject: [PATCH 13/42] test(apollo): assert the unparseable-directory warning, rename a loose test The warning capture apparatus asserted nothing, so deleting the warn in Apollo.pm would not have failed the suite. Assert the offending Apollo id appears -- the actionable content -- rather than the surrounding prose. Co-Authored-By: Claude Opus 5 --- Model/t/apollo.t | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Model/t/apollo.t b/Model/t/apollo.t index 42fdb3322f..2156a95285 100644 --- a/Model/t/apollo.t +++ b/Model/t/apollo.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 15; +use Test::More tests => 17; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Apollo; @@ -17,7 +17,9 @@ is($t->{annotation_count}, 10, 'annotation count carried'); is($t->{abbrev}, 'tgonME49', 'abbrev parsed from directory'); my $c = $live->{cneoJEC21}; -is($c->{annotation_count}, 14, 'the organism we must not lose is parsed'); +# The annotation count is the load-bearing fact for this organism: it is the +# number that makes deleting it expensive. +is($c->{annotation_count}, 14, 'cneoJEC21 carries its annotation count'); # commonName is curator-editable, so it is cross-checked, never matched on. is($A->commonNameDisagrees($live->{pfal3D7}, 'Plasmodium falciparum 3D7'), 1, @@ -70,6 +72,11 @@ like($@, qr/two organisms with directory .*dupe.*\n?.*ids 10 and 11/s, my @warnings; my $bad = do { local $SIG{__WARN__} = sub { push @warnings, $_[0] }; - $A->normalise([{id => 99, commonName => 'Z', directory => ''}]); + $A->normalise([{id => 9000003, commonName => 'Z', directory => ''}]); }; is_deeply($bad, {}, 'organism with an unparseable directory is skipped, not keyed as ""'); + +# The operator has to be able to act on this, which means knowing WHICH Apollo +# record is malformed. Assert the id appears, not the sentence around it. +is(scalar @warnings, 1, 'an unparseable directory produces exactly one warning'); +like($warnings[0], qr/9000003/, 'and the warning names the offending Apollo id'); From 57c5413785eb5820e7425a3aa3e723387e063732 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 20:56:37 -0400 Subject: [PATCH 14/42] refactor(apollo): collect Portal warnings instead of writing to stderr loadFromCommand treats any stderr byte from its child as proof the output is untrustworthy, while the module itself warned to stderr from the build-number skip path. A caller applying this module's own policy to it would read a handled skip as a release-breaking error. Warnings are now collected and exposed via warnings(), reset per normalise() call; the caller decides whether to print. Matches the pattern specified for ApolloRelease::Rename. Also: capture jbrowseOrganismList's stderr to a File::Temp rather than a fixed /tmp path (concurrent runs could truncate each other, and the size check is only meaningful if nothing else can write the file), keeping it on the failure paths for inspection; shell-quote the project name, which reaches the shell from a --project flag; die on a duplicate organism_abbrev rather than silently discarding an organism, as the sibling Apollo module does; and warn rather than die on a missing flag, since a NULL column should not stop a release. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Portal.pm | 83 ++++++++++++++++++++------ Model/t/portal.t | 78 ++++++++++++++++++------ 2 files changed, 125 insertions(+), 36 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Portal.pm b/Model/lib/perl/ApolloRelease/Portal.pm index 26e4aab19b..0623354cef 100644 --- a/Model/lib/perl/ApolloRelease/Portal.pm +++ b/Model/lib/perl/ApolloRelease/Portal.pm @@ -4,6 +4,7 @@ use strict; use warnings; use JSON; +use File::Temp; use Scalar::Util qw(looks_like_number); # Reads the organism list produced by Model/bin/jbrowseOrganismList and @@ -11,20 +12,50 @@ use Scalar::Util qw(looks_like_number); # strings "1"/"0". Everything downstream sees plain 0/1 and a single # `abbrev` key. +# Warnings are COLLECTED, never written to stderr. loadFromCommand treats any +# stderr byte from its child as proof that child's output is untrustworthy, and +# a caller applying that same policy to us -- the policy this module +# demonstrates -- would read a handled, recoverable skip as a release-breaking +# error. The caller decides whether to print these. +my @WARNINGS; + +sub warnings { return @WARNINGS } + sub loadFromCommand { my ($class, $projectName) = @_; my $gusHome = $ENV{GUS_HOME} or die "GUS_HOME is not set\n"; - my $cmd = "$gusHome/bin/jbrowseOrganismList $projectName"; - my $json = `$cmd 2>/tmp/apolloRelease.organismList.err`; - die "jbrowseOrganismList failed (exit " . ($? >> 8) . "); see /tmp/apolloRelease.organismList.err\n" - if $?; + # A private per-run capture file. A fixed path in /tmp lets two concurrent + # runs truncate or interleave each other's diagnostics, and the size check + # below is only meaningful if nothing else can write to the file. + my $errFile = File::Temp->new(TEMPLATE => 'apolloRelease.organismList.XXXXXXXX', + TMPDIR => 1, + SUFFIX => '.err'); + + # Shell-quoted rather than run through a list-form exec. We need the child's + # stdout captured AND its stderr diverted to a file we can stat afterwards, + # which one shell redirect does and a list-form open would need dup/restore + # gymnastics to match. $projectName arrives from a --project CLI flag, so a + # typo -- not merely malice -- can otherwise reach the shell as syntax. + my $quoted = "'" . ($projectName =~ s/'/'\\''/gr) . "'"; + my $cmd = "$gusHome/bin/jbrowseOrganismList $quoted"; + + my $json = `$cmd 2>'$errFile'`; + + # On the failure paths keep the capture file behind for inspection; on the + # happy path File::Temp removes it when $errFile goes out of scope. + if ($?) { + $errFile->unlink_on_destroy(0); + die "jbrowseOrganismList failed (exit " . ($? >> 8) . "); see $errFile\n"; + } - my $errSize = -s "/tmp/apolloRelease.organismList.err" || 0; - die "jbrowseOrganismList wrote $errSize bytes to stderr; refusing to trust its output.\n" - . "See /tmp/apolloRelease.organismList.err\n" - if $errSize; + my $errSize = -s "$errFile" || 0; + if ($errSize) { + $errFile->unlink_on_destroy(0); + die "jbrowseOrganismList wrote $errSize bytes to stderr; refusing to trust its output.\n" + . "See $errFile\n"; + } return $class->normalise(decode_json($json)); } @@ -46,21 +77,29 @@ sub loadFromFile { sub normalise { my ($class, $decoded) = @_; + @WARNINGS = (); + my %byAbbrev; foreach my $raw (@{$decoded->{organisms}}) { my $abbrev = $raw->{organism_abbrev}; + # Keying by abbrev means a duplicate would silently discard an organism. + # Dying matches the sibling Apollo module, and matches this module's own + # paranoia about a malformed flag. + die "duplicate organism_abbrev '$abbrev' in the organism list\n" + if exists $byAbbrev{$abbrev}; + $byAbbrev{$abbrev} = { abbrev => $abbrev, name => $raw->{name}, name_for_filenames => $raw->{name_for_filenames}, strain_abbrev => $raw->{strain_abbrev}, species_taxon => $raw->{species_ncbi_tax_id}, - is_reference => _boolean($raw->{is_reference_strain}), - is_annotated => _boolean($raw->{is_annotated_genome}), + is_reference => _boolean($raw->{is_reference_strain}, $abbrev, 'is_reference_strain'), + is_annotated => _boolean($raw->{is_annotated_genome}, $abbrev, 'is_annotated_genome'), history => $raw->{history} || [], - latest_annotation_version => $class->_latestAnnotationVersion($raw->{history}), + latest_annotation_version => $class->_latestAnnotationVersion($raw->{history}, $abbrev), }; } @@ -70,13 +109,21 @@ sub normalise { # is_reference_strain / is_annotated_genome arrive as the strings "1"/"0". # Coerce numerically: the string "0.0" is TRUE under Perl's boolean rules but # 0 numerically, and treating it as true would silently invert the flag. -# These are numeric flags in the database, so a non-numeric value means the -# upstream query changed shape -- fail rather than guess. +# +# The two bad-input paths differ ON PURPOSE. A MISSING flag is plausible +# upstream (a NULL column) and must not stop a release over one row, so it +# warns and reads as false. A flag that is PRESENT but non-numeric means the +# query changed shape; there is no safe reading of it, so it dies. sub _boolean { - my ($value) = @_; + my ($value, $abbrev, $field) = @_; + + unless (defined $value) { + push @WARNINGS, "$abbrev: $field is missing; treating it as false"; + return 0; + } - return 0 unless defined $value; - die "expected a numeric flag, got '$value'\n" unless looks_like_number($value); + die "$abbrev: expected a numeric flag for $field, got '$value'\n" + unless looks_like_number($value); return $value + 0 ? 1 : 0; } @@ -86,7 +133,7 @@ sub _boolean { # the organism's identity there. build_number is a string and is not always # an integer ("19.9" occurs), so compare numerically by intent. sub _latestAnnotationVersion { - my ($class, $history) = @_; + my ($class, $history, $abbrev) = @_; return undef unless $history && @$history; @@ -96,7 +143,7 @@ sub _latestAnnotationVersion { unless (looks_like_number($h->{build_number})) { my $shown = defined $h->{build_number} ? $h->{build_number} : '(undef)'; - warn "skipping history row with non-numeric build_number '$shown'\n"; + push @WARNINGS, "$abbrev: skipping history row with non-numeric build_number '$shown'"; next; } diff --git a/Model/t/portal.t b/Model/t/portal.t index 8533b49e58..732e3e1c9f 100644 --- a/Model/t/portal.t +++ b/Model/t/portal.t @@ -1,11 +1,25 @@ use strict; use warnings; -use Test::More tests => 13; +use Test::More tests => 19; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; my $P = 'ApiCommonModel::Model::ApolloRelease::Portal'; +# A synthetic organism carries filler that no test is about. Defaulting it +# here leaves each case showing only the fields it actually exercises. +sub _org { + my (%overrides) = @_; + return { name => 'x', + name_for_filenames => 'x', + strain_abbrev => 'x', + species_ncbi_tax_id => '1', + is_reference_strain => '1', + is_annotated_genome => '1', + history => [], + %overrides }; +} + my $orgs = $P->loadFromFile("Model/t/fixtures/portal.json"); is(ref($orgs), 'HASH', 'returns a hash keyed by abbrev'); @@ -26,24 +40,22 @@ is($P->qualifies($b), 0, 'non-reference organism does not qualify'); # "0.0" is truthy as a Perl string but zero numerically. Pin the coercion so a # future upstream change to the flag's format cannot silently invert it. my $synthetic = $P->normalise({organisms => [ - {organism_abbrev => 'zeroPointZero', is_reference_strain => '0.0', is_annotated_genome => '1', - name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', - history => [{build_number => '9', annotation_version => 'older'}, - {build_number => '19.9', annotation_version => 'middle'}, - {build_number => '71', annotation_version => 'newest'}]}, + _org(organism_abbrev => 'zeroPointZero', is_reference_strain => '0.0', + history => [{build_number => '9', annotation_version => 'older'}, + {build_number => '19.9', annotation_version => 'middle'}, + {build_number => '71', annotation_version => 'newest'}]), ]}); is($synthetic->{zeroPointZero}{is_reference}, 0, '"0.0" normalises to false, not true'); is($synthetic->{zeroPointZero}{latest_annotation_version}, 'newest', 'highest build number wins numerically: beats lexically-larger "9" and a decimal'); eval { $P->normalise({organisms => [ - {organism_abbrev => 'bogus', is_reference_strain => 'yes', is_annotated_genome => '1', history => []}, + _org(organism_abbrev => 'bogus', is_reference_strain => 'yes'), ]}) }; like($@, qr/expected a numeric flag/, 'a non-numeric flag is a hard error, not a guess'); -# A non-numeric build_number is skipped deliberately, with our own warning -- -# not left to Perl's numeric comparison, whose "isn't numeric" warning would -# go to stderr, which loadFromCommand treats as fatal. +# A non-numeric build_number is skipped deliberately rather than left to Perl's +# numeric comparison. # # Both organisms are needed to pin the skip. messyBuilds shows a bad row does # not corrupt the ordering of the good rows around it -- but that alone would @@ -52,15 +64,45 @@ like($@, qr/expected a numeric flag/, 'a non-numeric flag is a hard error, not a # guard the row is skipped and there is no latest version; without it the row # becomes $best by default and 'bogus' is returned as the annotation version. my $messy = $P->normalise({organisms => [ - {organism_abbrev => 'messyBuilds', is_reference_strain => '1', is_annotated_genome => '1', - name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', - history => [{build_number => 'not-a-number', annotation_version => 'bogus'}, - {build_number => '9', annotation_version => 'older'}, - {build_number => '71', annotation_version => 'newest'}]}, - {organism_abbrev => 'onlyBad', is_reference_strain => '1', is_annotated_genome => '1', - name => 'x', name_for_filenames => 'x', strain_abbrev => 'x', species_ncbi_tax_id => '1', - history => [{build_number => 'not-a-number', annotation_version => 'bogus'}]}, + _org(organism_abbrev => 'messyBuilds', + history => [{build_number => 'not-a-number', annotation_version => 'bogus'}, + {build_number => '9', annotation_version => 'older'}, + {build_number => '71', annotation_version => 'newest'}]), + _org(organism_abbrev => 'onlyBad', + history => [{build_number => 'not-a-number', annotation_version => 'bogus'}]), ]}); is(join('|', map { $messy->{$_}{latest_annotation_version} // 'undef' } qw(messyBuilds onlyBad)), 'newest|undef', 'a non-numeric build_number is skipped: it neither wins nor disturbs the good rows'); + +# A skip stays REPORTABLE, it just does not go to stderr -- this module's own +# loadFromCommand treats a child's stderr byte as fatal, so warning that way +# would make a handled skip look like a release-breaking error to a caller +# applying the same rule to us. +$P->normalise({organisms => [ + _org(organism_abbrev => 'messyBuilds', + history => [{build_number => 'not-a-number', annotation_version => 'bogus'}]), +]}); +my @skipWarnings = $P->warnings; +is(scalar(@skipWarnings), 1, 'warnings are reset per normalise call, not accumulated'); +like($skipWarnings[0], qr/messyBuilds.*non-numeric build_number 'not-a-number'/, + 'the collected warning names the organism it concerns'); + +$P->loadFromFile("Model/t/fixtures/portal.json"); +is(scalar($P->warnings), 0, 'a clean document collects no warnings'); + +# Asymmetry with the die-on-garbage path above is deliberate: a NULL flag is +# plausible upstream and must not stop a release over one row. +my $missing = $P->normalise({organisms => [ + _org(organism_abbrev => 'noFlag', is_reference_strain => undef), +]}); +is($missing->{noFlag}{is_reference}, 0, 'a missing flag reads as false rather than dying'); +like(join("\n", $P->warnings), qr/noFlag: is_reference_strain is missing/, + 'a missing flag is still reported, naming the organism and the field'); + +eval { $P->normalise({organisms => [ + _org(organism_abbrev => 'dupe'), + _org(organism_abbrev => 'dupe'), +]}) }; +like($@, qr/duplicate organism_abbrev 'dupe'/, + 'a duplicate abbrev dies rather than silently discarding an organism'); From 4cd7d6da5c50983814a8ef8367c3523eb6bca8f6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:01:28 -0400 Subject: [PATCH 15/42] feat(apollo): add reconciliation with add/prune/rename invariant Update and rename entries carry the organism's current publicMode so the command generator can echo it back; 17 live organisms are deliberately hidden and must not be re-published by a routine update. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Reconcile.pm | 230 ++++++++++++++++++++++ Model/t/reconcile.t | 133 +++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Reconcile.pm create mode 100644 Model/t/reconcile.t diff --git a/Model/lib/perl/ApolloRelease/Reconcile.pm b/Model/lib/perl/ApolloRelease/Reconcile.pm new file mode 100644 index 0000000000..5d11166731 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Reconcile.pm @@ -0,0 +1,230 @@ +package ApiCommonModel::Model::ApolloRelease::Reconcile; + +use strict; +use warnings; + +use ApiCommonModel::Model::ApolloRelease::Portal; + +# Pure set logic over the three inputs. Produces five buckets: +# +# update in Apollo and on the portal -> regenerate + update +# add_candidate qualifies, not in Apollo -> needs approval +# prune_candidate in Apollo, gone from the portal -> needs approval +# rename same genome under a new abbrev -> repoint in place +# exception in Apollo, fails the criteria -> no action, reported +# +# plus one report-only bucket: +# +# redundant_overlay an overlay line that changed nothing +# +# The roster is SEEDED FROM APOLLO. reference+annotated only ever proposes an +# add; it never removes anything. + +sub reconcile { + my ($class, $portal, $live, $overlay, $renames) = @_; + + $renames ||= {}; + + my %result = ( + update => [], + add_candidate => [], + prune_candidate => [], + rename => [], + exception => [], + redundant_overlay => [], + ); + + my %renameTarget = $class->_validateRenames($portal, $live, $renames); + + foreach my $abbrev (sort keys %$live) { + my $apollo = $live->{$abbrev}; + + if (exists $renames->{$abbrev}) { + my $newAbbrev = $renames->{$abbrev}; + my $target = $portal->{$newAbbrev}; + + push @{$result{rename}}, { + from_abbrev => $abbrev, + to_abbrev => $newAbbrev, + apollo_id => $apollo->{id}, + annotation_count => $apollo->{annotation_count}, + organism => $target, + # Same reason as `update` below: a rename must not change visibility. + public_mode => $apollo->{public_mode}, + }; + next; + } + + my $organism = $portal->{$abbrev}; + + unless ($organism) { + push @{$result{prune_candidate}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + common_name => $apollo->{common_name}, + annotation_count => $apollo->{annotation_count}, + approved => $overlay->{remove}{$abbrev} ? 1 : 0, + reason => $overlay->{remove}{$abbrev}, + }; + next; + } + + # In Apollo but failing the criteria: a decision made once, reported so + # nobody "fixes" it, and so a newly-demoted organism becomes visible. + unless (ApiCommonModel::Model::ApolloRelease::Portal->qualifies($organism)) { + push @{$result{exception}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + organism => $organism, + annotation_count => $apollo->{annotation_count}, + is_reference => $organism->{is_reference}, + is_annotated => $organism->{is_annotated}, + }; + next; + } + + push @{$result{update}}, { + abbrev => $abbrev, + apollo_id => $apollo->{id}, + organism => $organism, + # Echoed straight back in the update command. 17 organisms are hidden + # by curators; forcing publicMode=true would silently re-publish them. + public_mode => $apollo->{public_mode}, + }; + } + + foreach my $abbrev (sort keys %$portal) { + next if $live->{$abbrev}; + next if $renameTarget{$abbrev}; + next if $overlay->{remove}{$abbrev}; + + my $organism = $portal->{$abbrev}; + next unless ApiCommonModel::Model::ApolloRelease::Portal->qualifies($organism) + || $overlay->{add}{$abbrev}; + + push @{$result{add_candidate}}, { + abbrev => $abbrev, + organism => $organism, + approved => $overlay->{add}{$abbrev} ? 1 : 0, + reason => $overlay->{add}{$abbrev}, + }; + } + + $result{redundant_overlay} = $class->_redundantOverlay($portal, $live, $overlay); + + $class->assertInvariants(\%result); + + return \%result; +} + +# The rename map is the one input with no machine source: a human asserts that +# two abbrevs are the same genome. Every way of quietly ignoring one of those +# assertions loses something -- so all three incoherent shapes die here, before +# any bucket is built, rather than being discovered by whoever reads the +# generated commands. +sub _validateRenames { + my ($class, $portal, $live, $renames) = @_; + + my %renameTarget; + + foreach my $from (sort keys %$renames) { + my $to = $renames->{$from}; + + die "rename target for $from is empty\n" + unless defined $to && length $to; + + # A source that is not in Apollo has nothing to repoint. Dropping it + # silently is worse than it sounds: the target abbrev is still treated as + # "spoken for" and so is skipped by the add loop, leaving the organism + # neither renamed into nor added -- it simply disappears from the release. + die "rename source $from is not in Apollo; nothing to rename\n" + unless $live->{$from}; + + # Renaming onto an abbrev the portal does not know produces an Apollo + # organism pointing at a directory that will never be built. + die "rename target $to is not on the portal\n" + unless $portal->{$to}; + + # Two organisms repointed at one directory: Apollo would then hold two + # organisms sharing a directory, which the Apollo loader refuses to read + # at all on the next run. Catch it while it is still a text file. + die "two organisms renamed to $to ($renameTarget{$to} and $from)\n" + if exists $renameTarget{$to}; + + # An abbrev that is both a rename source and a rename target means the map + # describes a chain; there is no defensible order to apply it in. + die "$to is both a rename source and a rename target\n" + if exists $renames->{$to}; + + $renameTarget{$to} = $from; + } + + return %renameTarget; +} + +# An overlay line that changes nothing is not an error -- an organism approved +# last release is legitimately still listed -- but it is a recorded human +# decision the tool did nothing with, so it is reported rather than swallowed. +# Only true no-ops qualify: a `remove` naming an organism that is on the portal +# but not in Apollo really does suppress an add, so it is NOT redundant. +sub _redundantOverlay { + my ($class, $portal, $live, $overlay) = @_; + + my @redundant; + + foreach my $abbrev (sort keys %{$overlay->{add}}) { + push @redundant, { + abbrev => $abbrev, + directive => 'add', + reason => $overlay->{add}{$abbrev}, + note => $live->{$abbrev} ? 'already in Apollo' + : 'not on the portal', + } if $live->{$abbrev} || !$portal->{$abbrev}; + } + + foreach my $abbrev (sort keys %{$overlay->{remove}}) { + next if $live->{$abbrev} || $portal->{$abbrev}; + push @redundant, { + abbrev => $abbrev, + directive => 'remove', + reason => $overlay->{remove}{$abbrev}, + note => 'in neither Apollo nor the portal', + }; + } + + return \@redundant; +} + +# An add plus a prune for the same genome creates a new Apollo organism with no +# annotations and orphans the one holding the curation work -- from the same +# inputs and the same API as the correct behaviour. This is the failure mode +# that must never ship. +# +# reconcile() as written cannot reach either state: the add loop skips every +# rename target, and a live organism named as a rename source takes the rename +# branch and never falls through to prune. So this is not a guard on today's +# control flow -- it is a guard on tomorrow's. Both of those skips are one +# `next` each, in loops that will grow conditions, and the bug they prevent is +# invisible in the output (an add and a prune both look correct in isolation). +# It is a public class method so it can be, and is, tested at its own seam, and +# so a caller assembling a result by other means can assert on it. +sub assertInvariants { + my ($class, $result) = @_; + + my %renamedFrom = map { $_->{from_abbrev} => 1 } @{$result->{rename}}; + my %renamedTo = map { $_->{to_abbrev} => 1 } @{$result->{rename}}; + + foreach my $add (@{$result->{add_candidate}}) { + die "INVARIANT VIOLATED: $add->{abbrev} is both a rename target and an add candidate\n" + if $renamedTo{$add->{abbrev}}; + } + + foreach my $prune (@{$result->{prune_candidate}}) { + die "INVARIANT VIOLATED: $prune->{abbrev} is both a rename source and a prune candidate\n" + if $renamedFrom{$prune->{abbrev}}; + } + + return 1; +} + +1; diff --git a/Model/t/reconcile.t b/Model/t/reconcile.t new file mode 100644 index 0000000000..886c06ef90 --- /dev/null +++ b/Model/t/reconcile.t @@ -0,0 +1,133 @@ +use strict; +use warnings; +use Test::More tests => 26; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Portal; +use ApiCommonModel::Model::ApolloRelease::Apollo; +use ApiCommonModel::Model::ApolloRelease::Overlay; +use ApiCommonModel::Model::ApolloRelease::Reconcile; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Reconcile'; + +my $portal = ApiCommonModel::Model::ApolloRelease::Portal->loadFromFile("Model/t/fixtures/portal.json"); +my $live = ApiCommonModel::Model::ApolloRelease::Apollo->loadFromFile("Model/t/fixtures/apollo.json"); +my $overlay = ApiCommonModel::Model::ApolloRelease::Overlay->parseString("remove hsapREF # host genome\n"); + +# No renames supplied: cneoJEC21 and cglaCBS138 are unmatched, so both are prunes. +my $r = $R->reconcile($portal, $live, $overlay, {}); + +my %update = map { $_->{abbrev} => 1 } @{$r->{update}}; +ok($update{tgonME49}, 'in Apollo and on portal -> update'); +ok($update{pfal3D7}, 'GUI-renamed organism still updates (matched on directory)'); + +my %prune = map { $_->{abbrev} => 1 } @{$r->{prune_candidate}}; +ok($prune{cneoJEC21}, 'in Apollo, absent from portal -> prune candidate'); +ok($prune{cglaCBS138}, 'second orphan is a prune candidate'); + +my ($cneo) = grep { $_->{abbrev} eq 'cneoJEC21' } @{$r->{prune_candidate}}; +is($cneo->{annotation_count}, 14, 'prune candidates carry their annotation count'); + +my %add = map { $_->{abbrev} => 1 } @{$r->{add_candidate}}; +ok($add{hcapNAm1}, 'qualifying organism absent from Apollo -> add candidate'); +ok($add{cdenJEC21}, 'rename target is an add candidate when no rename is detected'); +ok(!$add{hsapREF}, 'overlay remove suppresses an add candidate'); + +my %exception = map { $_->{abbrev} => 1 } @{$r->{exception}}; +ok($exception{tbruLister427_2018}, 'in Apollo but not reference+annotated -> exception'); +ok(!$update{tbruLister427_2018}, 'an exception is not also an update'); + +# Now supply the rename, and the pair must collapse. +my $renames = { cneoJEC21 => 'cdenJEC21' }; +my $r2 = $R->reconcile($portal, $live, $overlay, $renames); + +my %prune2 = map { $_->{abbrev} => 1 } @{$r2->{prune_candidate}}; +my %add2 = map { $_->{abbrev} => 1 } @{$r2->{add_candidate}}; +ok(!$prune2{cneoJEC21} && !$add2{cdenJEC21}, + 'a detected rename removes both the prune and the add'); + +is(scalar(@{$r2->{rename}}), 1, 'and produces exactly one rename entry'); + +my $overlayWithPrune = ApiCommonModel::Model::ApolloRelease::Overlay->parseString( + "remove cglaCBS138 # renamed to nglaCBS138, zero annotations\n"); +my $r3 = $R->reconcile($portal, $live, $overlayWithPrune, {}); +my ($cgla) = grep { $_->{abbrev} eq 'cglaCBS138' } @{$r3->{prune_candidate}}; +is($cgla->{approved}, 1, 'an overlay remove approves the prune'); + +# --------------------------------------------------------------------------- +# publicMode passthrough. 17 live organisms are deliberately hidden; an update +# or a rename that forces publicMode=true silently re-publishes them. +# --------------------------------------------------------------------------- + +my ($tgon) = grep { $_->{abbrev} eq 'tgonME49' } @{$r->{update}}; +is($tgon->{public_mode}, 1, 'update carries the live publicMode through'); + +my ($ren) = @{$r2->{rename}}; +is($ren->{public_mode}, 1, 'rename carries the live publicMode through'); +is($ren->{annotation_count}, 14, 'rename carries the curation work it is protecting'); + +# --------------------------------------------------------------------------- +# Attacking the invariant directly. reconcile() cannot construct a violating +# result (see Reconcile.pm), so the guard is exercised at its own seam: it is a +# public class method precisely so that it is not dead, untested code. +# --------------------------------------------------------------------------- + +eval { + $R->assertInvariants({ + update => [], + add_candidate => [{ abbrev => 'cdenJEC21' }], + prune_candidate => [], + rename => [{ from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21' }], + exception => [], + }); +}; +like($@, qr/INVARIANT VIOLATED: cdenJEC21 is both a rename target and an add candidate/, + 'guard fires when a rename target is also an add candidate'); + +eval { + $R->assertInvariants({ + update => [], + add_candidate => [], + prune_candidate => [{ abbrev => 'cneoJEC21' }], + rename => [{ from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21' }], + exception => [], + }); +}; +like($@, qr/INVARIANT VIOLATED: cneoJEC21 is both a rename source and a prune candidate/, + 'guard fires when a rename source is also a prune candidate'); + +ok($R->assertInvariants($r2), 'a real reconcile result passes the guard'); + +# --------------------------------------------------------------------------- +# Incoherent rename input. A rename map is a human statement of intent; every +# way of ignoring one silently loses curation work or an organism. +# --------------------------------------------------------------------------- + +eval { $R->reconcile($portal, $live, $overlay, { zzzNotLive => 'hcapNAm1' }) }; +like($@, qr/rename source zzzNotLive is not in Apollo/, + 'a rename whose source is not live dies instead of vanishing'); + +eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => 'cdenJEC21', + cglaCBS138 => 'cdenJEC21' }) }; +like($@, qr/two organisms renamed to cdenJEC21/, + 'two sources claiming one target is refused'); + +eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => 'notOnPortal' }) }; +like($@, qr/rename target notOnPortal is not on the portal/, + 'a rename target absent from the portal dies'); + +# --------------------------------------------------------------------------- +# Overlay hygiene. A no-op overlay line is not an error, but it must be +# reported: it is a human decision the tool did nothing with. +# --------------------------------------------------------------------------- + +is(scalar(@{$r->{redundant_overlay}}), 0, 'a fully-effective overlay reports nothing redundant'); + +my $noisyOverlay = ApiCommonModel::Model::ApolloRelease::Overlay->parseString( + "add tgonME49 # already live, left over from last release\n" + . "remove zzzGhost # organism nobody has heard of\n"); +my $r4 = $R->reconcile($portal, $live, $noisyOverlay, {}); +my %redundant = map { $_->{abbrev} => $_ } @{$r4->{redundant_overlay}}; +is($redundant{tgonME49}{directive}, 'add', 'an overlay add for a live organism is reported as redundant'); +is($redundant{zzzGhost}{directive}, 'remove', 'an overlay entry naming nothing at all is reported'); +is($redundant{tgonME49}{reason}, 'already live, left over from last release', + 'the redundant entry carries its reason so the file can be cleaned up'); From 50fdc30099e3b36431954d1aa21a5cfd29c5b9c0 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:02:35 -0400 Subject: [PATCH 16/42] Apollo plan: report the redundant_overlay bucket Reconcile now reports overlay entries that no longer have any effect -- an add for an organism already live, or a remove naming an abbrev in neither Apollo nor the portal. Fatal would break a release over file hygiene; silence would discard a recorded human decision. Co-Authored-By: Claude Opus 5 --- .../plans/2026-08-21-apollo-release-package.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-08-21-apollo-release-package.md b/docs/superpowers/plans/2026-08-21-apollo-release-package.md index f90a8a8ff8..9b75129992 100644 --- a/docs/superpowers/plans/2026-08-21-apollo-release-package.md +++ b/docs/superpowers/plans/2026-08-21-apollo-release-package.md @@ -1822,7 +1822,7 @@ Create `Model/t/report.t`: ```perl use strict; use warnings; -use Test::More tests => 5; +use Test::More tests => 6; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Report; @@ -1835,6 +1835,8 @@ my $result = { {abbrev => 'zzzRisky', annotation_count => 14, common_name => 'Something Curated'} ], rename => [ {from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', annotation_count => 14} ], exception => [ {abbrev => 'tbruLister427_2018', is_reference => 0, is_annotated => 1} ], + redundant_overlay => [ {abbrev => 'staleEntry', directive => 'add', reason => 'approved 2025', + note => 'already in Apollo'} ], }; my $text = $R->render($result, {build => 71, environment => 'prod'}); @@ -1844,6 +1846,7 @@ like($text, qr/cneoJEC21.*cdenJEC21/, 'names both sides of a rename'); like($text, qr/zzzRisky.*14/, 'prune candidates show their annotation count'); like($text, qr/ANNOTATIONS WILL BE HIDDEN/, 'an annotated prune is called out loudly'); unlike($text, qr/cglaCBS138.*ANNOTATIONS WILL BE HIDDEN/, 'a zero-annotation prune is not'); +like($text, qr/staleEntry.*already in Apollo/, 'a redundant overlay entry is reported so the file can be tidied'); ``` - [ ] **Step 2: Run it and watch it fail** @@ -1878,7 +1881,7 @@ sub render { $context->{build} // '?', $context->{environment} // '?'); push @out, "summary\n"; - foreach my $bucket (qw(update add_candidate prune_candidate rename exception)) { + foreach my $bucket (qw(update add_candidate prune_candidate rename exception redundant_overlay)) { push @out, sprintf(" %-18s %d\n", $bucket, scalar @{$result->{$bucket} || []}); } push @out, "\n"; @@ -1913,6 +1916,17 @@ sub render { push @out, "\n"; } + # Report-only. An overlay line that no longer does anything is a decision + # that has been overtaken by events; printing it is how the file gets tidied + # instead of accreting entries nobody dares delete. + if (@{$result->{redundant_overlay} || []}) { + push @out, "redundant overlay entries -- no longer have any effect\n"; + foreach my $e (sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{redundant_overlay}}) { + push @out, sprintf(" %-26s %-7s %s\n", $e->{abbrev}, $e->{directive}, $e->{note}); + } + push @out, "\n"; + } + if (@{$result->{exception} || []}) { push @out, "exceptions -- in Apollo but not reference+annotated; no action\n"; foreach my $e (sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{exception}}) { From 76d88031471edd0b5a80de38418fcfa1e0d99a5f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:05:40 -0400 Subject: [PATCH 17/42] fix(apollo): trim whitespace from directory, report a non-JSON body as itself A stray trailing byte on Apollo's directory field yielded the abbrev 'tgonME49 ', which matches no portal organism -- reconciliation would then report the real genome as an add and the tainted one as a prune, the add-plus-prune-for-one-genome case this project exists to prevent. Trim surrounding whitespace along with trailing slashes, and validate the abbrev's shape so interior junk is skipped rather than admitted. Guard decode_json so an HTML login page served with a 200 status reports its source and likely cause instead of a bare 'malformed JSON string'. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Apollo.pm | 51 ++++++++++++++++++++++++-- Model/t/apollo.t | 39 +++++++++++++++++++- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Apollo.pm b/Model/lib/perl/ApolloRelease/Apollo.pm index 80b16d55d3..332f86db5c 100644 --- a/Model/lib/perl/ApolloRelease/Apollo.pm +++ b/Model/lib/perl/ApolloRelease/Apollo.pm @@ -31,7 +31,7 @@ sub loadFromApi { die "Apollo API request failed: " . $response->status_line . "\n" unless $response->is_success; - my $decoded = decode_json($response->content); + my $decoded = $class->_decodeRoster($response->content, $url); # An empty roster would make every organism look new, and the generated # commands would try to re-add the entire set. Fail instead. @@ -51,7 +51,28 @@ sub loadFromFile { my $json = <$fh>; close $fh; - return $class->normalise(decode_json($json)); + return $class->normalise($class->_decodeRoster($json, $path)); +} + +# A 200 carrying an HTML login page or a truncated body would otherwise die +# with a bare "malformed JSON string" from inside the JSON module, naming +# neither the source nor the likely cause. This project has already been +# bitten by exactly that shape: an unauthenticated request served a login page +# with a 200 status. +sub _decodeRoster { + my ($class, $body, $source) = @_; + + my $decoded = eval { decode_json($body) }; + + die "Apollo returned a non-JSON body from $source.\n" + . "This is usually an HTML login or error page served with a 200 status:\n" + . "check APOLLO_API_USER/PASS and network access to the API.\n" + unless $decoded; + + die "Apollo response from $source was not a list of organisms.\n" + unless ref $decoded eq 'ARRAY'; + + return $decoded; } # Public because it is the seam: it takes an already-decoded document, so the @@ -64,15 +85,37 @@ sub normalise { foreach my $raw (@$decoded) { my $directory = $raw->{directory} || ''; - $directory =~ s{/+$}{}; + + # Trim surrounding whitespace as well as trailing slashes, in one pass so + # that a mixture ("/data/apollo_data/tgonME49 /") is fully removed. A + # single stray trailing byte is not cosmetic here: it yields the abbrev + # "tgonME49 ", which matches no portal organism, so reconciliation reports + # the real genome as an add and the space-tainted one as a prune -- the + # add-plus-prune-for-one-genome case this project exists to prevent. + # "Machine-written by our own update commands" makes it unlikely, not + # impossible; an upstream concatenation bug is exactly how it would appear. + $directory =~ s{^\s+}{}; + $directory =~ s{[\s/]+$}{}; my ($abbrev) = $directory =~ m{([^/]+)$}; - unless ($abbrev) { + unless (defined $abbrev && length $abbrev) { warn "Apollo organism id $raw->{id} has an unparseable directory '$raw->{directory}'; skipping\n"; next; } + # Interior junk cannot be trimmed away without inventing an identity, so + # validate the shape instead. An abbrev containing a space or a newline + # would sail through the trim above and produce the same add-plus-prune + # pair. Skipping is the safe failure: an organism absent from this hash + # can only become an approval-gated add_candidate downstream, never a + # prune, because prune requires presence in Apollo. + unless ($abbrev =~ m{\A[A-Za-z0-9_.-]+\z}) { + warn "Apollo organism id $raw->{id} has a malformed abbrev '$abbrev' " + . "from directory '$raw->{directory}'; skipping\n"; + next; + } + # Two Apollo organisms sharing a directory is corruption, not a shape we # can normalise. Overwriting silently would drop one of them from the # roster, and the diff against the portal would then generate commands diff --git a/Model/t/apollo.t b/Model/t/apollo.t index 2156a95285..3a81b362ba 100644 --- a/Model/t/apollo.t +++ b/Model/t/apollo.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 17; +use Test::More tests => 24; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Apollo; @@ -80,3 +80,40 @@ is_deeply($bad, {}, 'organism with an unparseable directory is skipped, not keye # record is malformed. Assert the id appears, not the sentence around it. is(scalar @warnings, 1, 'an unparseable directory produces exactly one warning'); like($warnings[0], qr/9000003/, 'and the warning names the offending Apollo id'); + +# A single stray trailing byte is not cosmetic: the abbrev "tgonME49 " matches +# no portal organism, so reconciliation would report the real genome as an add +# and the tainted one as a prune -- add-plus-prune for one genome. +my $spaced = $A->normalise([ + {id => 3, commonName => 'Y', directory => '/data/apollo_data/tgonME49 '}, +]); +is_deeply([keys %$spaced], ['tgonME49'], 'trailing whitespace on directory is stripped'); + +my $both = $A->normalise([ + {id => 4, commonName => 'Y', directory => " /data/apollo_data/tgonME49 /\n"}, +]); +is_deeply([keys %$both], ['tgonME49'], 'mixed trailing whitespace and slashes are stripped'); + +# Interior junk cannot be trimmed without inventing an identity, so the shape +# is validated instead. Skipping is the safe failure: an absent organism can +# only become an approval-gated add_candidate, never a prune. +my @junkWarnings; +my $junk = do { + local $SIG{__WARN__} = sub { push @junkWarnings, $_[0] }; + $A->normalise([{id => 9000004, commonName => 'Z', directory => '/data/apollo_data/tgon ME49'}]); +}; +is_deeply($junk, {}, 'an abbrev with an interior space is skipped, not admitted'); +is(scalar @junkWarnings, 1, 'the malformed abbrev produces exactly one warning'); +like($junkWarnings[0], qr/9000004/, 'and the warning names the offending Apollo id'); + +# A 200 carrying an HTML login page must report as itself, not as a bare +# "malformed JSON string" from inside the JSON module. +eval { $A->_decodeRoster('Please log in', 'https://apollo.example') }; +like($@, qr{non-JSON body from https://apollo\.example}, 'a non-JSON body names its source'); + +eval { $A->_decodeRoster('{"error":"nope"}', 'https://apollo.example') }; +like($@, qr/not a list of organisms/, 'a JSON object that is not a roster is rejected'); + +# A still-set $@ at exit becomes the process exit status; clear it so a passing +# run exits 0. +$@ = ''; From 3e3c4fddc9a3533cb52b368e92fdb1f46adafdd4 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:10:42 -0400 Subject: [PATCH 18/42] fix(apollo): sharper overlay parse errors, cite both lines, hermetic test Parse failures now name the actual defect (missing organism vs missing reason vs unparseable) instead of always blaming the reason. Duplicate and contradiction errors cite the earlier line as well as the current one. The shipped-overlay test reads the repo copy, so it no longer depends on a prior bld into a shared GUS_HOME. Return shape unchanged: {add,remove}{abbrev} stays a reason string, which Reconcile.pm prints directly. Line numbers live in a private hash. Co-Authored-By: Claude Opus 5 --- Model/data/apollo/roster-overlay.txt | 29 +++++++----- Model/lib/perl/ApolloRelease/Overlay.pm | 63 +++++++++++++++++++++---- Model/t/overlay.t | 63 ++++++++++++++++++++----- 3 files changed, 120 insertions(+), 35 deletions(-) diff --git a/Model/data/apollo/roster-overlay.txt b/Model/data/apollo/roster-overlay.txt index 3f72288cd9..10c5214ac6 100644 --- a/Model/data/apollo/roster-overlay.txt +++ b/Model/data/apollo/roster-overlay.txt @@ -11,18 +11,21 @@ # Apollo curates pathogens. The previous script excluded these implicitly, # via an @databases array that omitted HostDB and SchistoDB. On the UniDB # portal that filter does not exist, so the exclusion has to be stated. +# +# All 11 seeded 2026-08-21 to reproduce b68 behaviour; each line names its +# organism rather than repeating that provenance. -remove hsapREF # host genome (seeded 2026-08-21 from b68 behaviour) -remove mmusC57BL6J # host genome (seeded 2026-08-21 from b68 behaviour) -remove rnorBNNHsdMcwi # host genome (seeded 2026-08-21 from b68 behaviour) -remove btauHereford # host genome (seeded 2026-08-21 from b68 behaviour) -remove clupfamiliarisSID07034 # host genome (seeded 2026-08-21 from b68 behaviour) -remove ggalbGalGal1 # host genome (seeded 2026-08-21 from b68 behaviour) -remove cpor2N # host genome (seeded 2026-08-21 from b68 behaviour) -remove mfasREF # host genome (seeded 2026-08-21 from b68 behaviour) -remove mmulAG07107 # host genome (seeded 2026-08-21 from b68 behaviour) -remove mmyomMyoMyo1 # host genome (seeded 2026-08-21 from b68 behaviour) -remove dmeliso-1 # host genome (seeded 2026-08-21 from b68 behaviour) +remove hsapREF # host genome: Homo sapiens +remove mmusC57BL6J # host genome: Mus musculus C57BL/6J +remove rnorBNNHsdMcwi # host genome: Rattus norvegicus BN/NHsdMcwi +remove btauHereford # host genome: Bos taurus Hereford +remove clupfamiliarisSID07034 # host genome: Canis lupus familiaris +remove ggalbGalGal1 # host genome: Gallus gallus bGalGal1 +remove cpor2N # host genome: Cavia porcellus 2N +remove mfasREF # host genome: Macaca fascicularis +remove mmulAG07107 # host genome: Macaca mulatta AG07107 +remove mmyomMyoMyo1 # host genome: Myotis myotis mMyoMyo1 +remove dmeliso-1 # host genome: Drosophila melanogaster iso-1 # --- Model fungi -------------------------------------------------------- # In FungiDB, qualify on the criteria, never been in Apollo. Seeded to match @@ -35,4 +38,6 @@ remove spom972h # model organism, never in Apollo; decision unco # --- Deliberately NOT listed here --------------------------------------- # hcapNAm1 (Histoplasma mississippiense NAm1) is a FungiDB pathogen that # qualifies and is absent from Apollo. It is left in the add-candidate bucket -# for the curation team rather than suppressed here. +# for the curation team rather than suppressed here. Model/t/overlay.t pins +# that it appears in neither direction, so adding it here without revisiting +# this comment fails the build. diff --git a/Model/lib/perl/ApolloRelease/Overlay.pm b/Model/lib/perl/ApolloRelease/Overlay.pm index 47d466ce59..3573ef45e1 100644 --- a/Model/lib/perl/ApolloRelease/Overlay.pm +++ b/Model/lib/perl/ApolloRelease/Overlay.pm @@ -11,11 +11,18 @@ use warnings; # The reason is mandatory. This file exists to record provenance; a line # without it is indistinguishable from the hardcoded __DATA__ block this # replaces, which froze two builds ago and drifted silently. +# +# parseString returns { add => {abbrev => reason}, remove => {abbrev => reason} }. +# The values are plain reason STRINGS and must stay that way: Reconcile.pm +# passes them straight into curator-facing report output, so wrapping them in +# a richer structure would print ARRAY(0x...) into a report and fail silently. +# Line numbers are therefore tracked in a private hash used only for errors. sub parseFile { my ($class, $path) = @_; - open(my $fh, '<', $path) or die "Cannot read roster overlay $path: $!"; + open(my $fh, '<:encoding(UTF-8)', $path) + or die "Cannot read roster overlay $path: $!\n"; local $/; my $text = <$fh>; close $fh; @@ -27,6 +34,12 @@ sub parseString { my ($class, $text) = @_; my %overlay = (add => {}, remove => {}); + + # Where each accepted entry came from, so a duplicate or a contradiction can + # name BOTH lines. Deliberately not part of the returned structure -- see + # the note at the top of the file. + my %lineOf; + my $lineNumber = 0; foreach my $line (split /\n/, $text) { @@ -35,34 +48,64 @@ sub parseString { next if $line =~ /^\s*$/; next if $line =~ /^\s*#/; + # (1) directive (2) abbrev (3) reason, to end of line my ($directive, $abbrev, $reason) = $line =~ /^\s*(\S+)\s+(\S+)\s*#\s*(\S.*?)\s*$/; - unless (defined $directive) { - my ($lone) = $line =~ /^\s*(\S+)\s*$/; - die "roster overlay line $lineNumber: missing organism or reason: '$line'\n" - if $lone; - die "roster overlay line $lineNumber: every entry needs a non-empty '# reason': '$line'\n"; - } + $class->_explainParseFailure($lineNumber, $line) unless defined $directive; die "roster overlay line $lineNumber: unknown directive '$directive' (expected add or remove)\n" unless $directive eq 'add' || $directive eq 'remove'; my $other = $directive eq 'add' ? 'remove' : 'add'; - die "roster overlay line $lineNumber: $abbrev is listed as both add and remove\n" + + die "roster overlay: $abbrev already appears as '$other' on line $lineOf{$other}{$abbrev}; " + . "line $lineNumber lists it as '$directive' -- an organism cannot be both\n" if exists $overlay{$other}{$abbrev}; # A repeat of the same directive would silently discard the first line's # reason. This file's whole purpose is provenance, so losing one is a # defect, not a merge: refuse it and make the author reconcile the two. - die "roster overlay line $lineNumber: $abbrev is listed as '$directive' more than once; " - . "merge the reasons into one line\n" + die "roster overlay: $abbrev already appears as '$directive' on line $lineOf{$directive}{$abbrev}; " + . "merge the reason from line $lineNumber into that line\n" if exists $overlay{$directive}{$abbrev}; $overlay{$directive}{$abbrev} = $reason; + $lineOf{$directive}{$abbrev} = $lineNumber; } return \%overlay; } +# Always dies. Names the actual defect where it is identifiable, because +# "needs a reason" is actively misleading on a line that has one and is +# missing the organism instead. +sub _explainParseFailure { + my ($class, $lineNumber, $line) = @_; + + my $prefix = "roster overlay line $lineNumber:"; + + # No '#' at all: whatever else is wrong, the missing reason is the headline, + # since the reason is the one thing this format exists to require. + unless ($line =~ /#/) { + my ($lone) = $line =~ /^\s*(\S+)\s*$/; + die "$prefix missing organism and reason: '$line'\n" if $lone; + die "$prefix every entry needs a '# reason': '$line'\n"; + } + + # There is a '#'. Count the tokens before it to name the defect. + my ($before) = $line =~ /^(.*?)#/; + my @tokens = split ' ', $before; + + die "$prefix missing organism: '$line'\n" + if @tokens == 1; + + # Two tokens plus a '#' is the shape the main regex accepts, so reaching + # here means the reason itself was empty or all whitespace. + die "$prefix every entry needs a non-empty '# reason': '$line'\n" + if @tokens == 2; + + die "$prefix could not parse '$line' as ' # '\n"; +} + 1; diff --git a/Model/t/overlay.t b/Model/t/overlay.t index 9f352bc3c4..0dae1a29af 100644 --- a/Model/t/overlay.t +++ b/Model/t/overlay.t @@ -1,6 +1,7 @@ use strict; use warnings; -use Test::More tests => 12; +use Test::More tests => 18; +use FindBin; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Overlay; @@ -23,13 +24,34 @@ like($overlay->{add}{someNewOrg}, qr/Uli/, 'approver retained'); eval { $O->parseString("bogus hsapREF # what\n") }; like($@, qr/unknown directive 'bogus'/, 'unknown directive rejected'); -eval { $O->parseString("add hsapREF # x\nremove hsapREF # y\n") }; -like($@, qr/both add and remove/, 'contradictory entry rejected'); +# Both offending lines are named. Finding the other occurrence by hand in a +# file being edited under time pressure is exactly the cost this avoids. +eval { $O->parseString("# header\nadd hsapREF # x\n\nremove hsapREF # y\n") }; +like($@, qr/hsapREF already appears as 'add' on line 2; line 4 lists it as 'remove'.*cannot be both/, + 'contradictory entry rejected, citing both lines'); eval { $O->parseString("add\n") }; like($@, qr/missing organism/, 'malformed line rejected'); -# --- gaps considered in review ----------------------------------------- +# --- parse-failure branches -------------------------------------------- +# +# The fallback used to claim "needs a reason" for every unparseable line, +# including lines that plainly had one. Each branch below is a defect the +# message can identify; anything else gets a defect-agnostic message rather +# than a confident wrong one. + +eval { $O->parseString("add # approved by Uli\n") }; +like($@, qr/missing organism/, 'a reason with no organism is reported as a missing organism'); +unlike($@, qr/reason/, '... and does not claim the reason is missing, because it is not'); + +eval { $O->parseString("remove hsapREF\n") }; +like($@, qr/every entry needs a '# reason'/, 'an entry with no reason at all is reported as such'); + +eval { $O->parseString("add someOrg extra # approved by Uli\n") }; +like($@, qr/could not parse .* as ' # '/, + 'a stray third token gets the defect-agnostic message, not a wrong diagnosis'); + +# --- other gaps considered in review ----------------------------------- # BEHAVIOUR CHANGE (deliberate): the same organism repeated with the SAME # directive used to let the second line silently overwrite the first, throwing @@ -38,7 +60,8 @@ like($@, qr/missing organism/, 'malformed line rejected'); # prevent -- and two lines for one organism means two people wrote down two # different justifications, which a human must reconcile. Now a hard error. eval { $O->parseString("remove hsapREF # host genome\nremove hsapREF # not loaded here\n") }; -like($@, qr/more than once/, 'duplicate same-directive entry rejected, not silently merged'); +like($@, qr/hsapREF already appears as 'remove' on line 1; merge the reason from line 2/, + 'duplicate same-directive entry rejected, citing both lines'); # BEHAVIOUR CHANGE (deliberate): the reason group was `(.+?)` with `\s*` on # both sides, which happily matched an empty string -- "add x # " parsed and @@ -54,18 +77,32 @@ my $hashy = $O->parseString("add someOrg # approved by Uli, see ticket #4412 # d is($hashy->{add}{someOrg}, 'approved by Uli, see ticket #4412 # dup marker', 'a # inside the reason is kept verbatim, not treated as a second comment'); +# This file is edited on Macs and Windows. Both of these already parse, but +# only because the trailing `\s*` happens to absorb the `\r` and `\s` happens +# to cover tabs -- pin them so a future regex edit cannot silently break a +# whole platform's worth of edits. +my $crlf = $O->parseString("remove hsapREF # host genome\r\nadd someOrg # approved\r\n"); +is(join('|', $crlf->{remove}{hsapREF}, $crlf->{add}{someOrg}), 'host genome|approved', + 'CRLF line endings parse, with no stray carriage return in the reason'); + +my $tabs = $O->parseString("remove\thsapREF\t# host genome\n"); +is($tabs->{remove}{hsapREF}, 'host genome', 'tab-delimited lines parse'); + # The overlay path is passed in by a caller, so a typo or a missed `bld` is the # likely failure. Pin that the error names the path and the file's role rather # than dying somewhere later on an undef. eval { $O->parseFile("/nonexistent/apollo/roster-overlay.txt") }; -like($@, qr{Cannot read roster overlay /nonexistent/apollo/roster-overlay\.txt}, - 'a missing overlay file gives an actionable error naming the path'); +like($@, qr{^Cannot read roster overlay /nonexistent/apollo/roster-overlay\.txt: [^\n]+\n$}, + 'a missing overlay file gives an actionable error naming the path, with no perl line noise'); # Parsing the real shipped file is the only test that can catch a seeded line -# that stopped being parseable; it also pins that hcapNAm1 stays an add -# candidate rather than being quietly suppressed. -my $shipped = $O->parseFile($ENV{GUS_HOME} . "/data/ApiCommonModel/Model/apollo/roster-overlay.txt"); +# that stopped being parseable. Read it from the REPO, not from GUS_HOME: it +# is a plain data file copied verbatim by the build, so nothing is lost, and +# reading the installed copy would make this the one test that needs a prior +# `bld` -- and a stale artifact in a shared GUS_HOME would pass misleadingly. +my $shipped = $O->parseFile("$FindBin::Bin/../data/apollo/roster-overlay.txt"); is(join('/', scalar(keys %{$shipped->{remove}}), scalar(keys %{$shipped->{add}}), - ($shipped->{remove}{hcapNAm1} ? 'suppressed' : 'candidate')), - '13/0/candidate', - 'the shipped overlay parses: 13 removes, 0 adds, hcapNAm1 left as an add candidate'); + ($shipped->{remove}{hcapNAm1} ? 'suppressed' : 'candidate'), + ($shipped->{add}{hcapNAm1} ? 'forced' : 'candidate')), + '13/0/candidate/candidate', + 'the shipped overlay parses: 13 removes, 0 adds, hcapNAm1 neither suppressed nor force-added'); From a0c870cb1e14d19d5f42db9f2b7f9717025345d6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:16:54 -0400 Subject: [PATCH 19/42] test(apollo): cover hidden publicMode, rename dies, rename-shadowed overlay Add treeQM6a (real prod data: publicMode false, 11 annotations, on the portal) to both fixtures. Every previous publicMode assertion used a visible organism, so hardcoding public_mode => 1 -- the bug that would re-publish 17 deliberately hidden organisms -- passed all 26 tests. Both the update and rename paths are now mutation-checked. Also: refuse a rename onto an abbrev Apollo already holds (a merge, not a rename, and the same duplicate-directory corruption the two-sources check prevents), and consult the rename map when reporting redundant overlay lines -- a rename consumes both abbrevs and silently disarmed a line naming either end. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Reconcile.pm | 42 +++++++++++++++---- Model/t/apollo.t | 2 +- Model/t/fixtures/apollo.json | 6 ++- Model/t/fixtures/portal.json | 43 ++++++++++++++++++++ Model/t/reconcile.t | 49 ++++++++++++++++++++++- 5 files changed, 131 insertions(+), 11 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Reconcile.pm b/Model/lib/perl/ApolloRelease/Reconcile.pm index 5d11166731..2387e704af 100644 --- a/Model/lib/perl/ApolloRelease/Reconcile.pm +++ b/Model/lib/perl/ApolloRelease/Reconcile.pm @@ -110,7 +110,8 @@ sub reconcile { }; } - $result{redundant_overlay} = $class->_redundantOverlay($portal, $live, $overlay); + $result{redundant_overlay} = + $class->_redundantOverlay($portal, $live, $overlay, $renames, \%renameTarget); $class->assertInvariants(\%result); @@ -156,6 +157,13 @@ sub _validateRenames { die "$to is both a rename source and a rename target\n" if exists $renames->{$to}; + # Repointing onto an abbrev Apollo already holds is a merge, not a rename: + # it would leave two Apollo organisms sharing one directory -- the same + # corruption the two-sources check above prevents, reached from the other + # side. Whichever curation set loses the coin toss is orphaned. + die "rename target $to is already in Apollo; that is a merge, not a rename\n" + if $live->{$to}; + $renameTarget{$to} = $from; } @@ -165,30 +173,48 @@ sub _validateRenames { # An overlay line that changes nothing is not an error -- an organism approved # last release is legitimately still listed -- but it is a recorded human # decision the tool did nothing with, so it is reported rather than swallowed. -# Only true no-ops qualify: a `remove` naming an organism that is on the portal +# +# The rename map has to be consulted here, not just $live and $portal: a rename +# consumes BOTH abbrevs of the pair, so it silently disarms an overlay line +# naming either end. Neither is visible in any other bucket -- the add loop +# skips a rename target outright, and a rename source never becomes a prune +# candidate, so its `approved` flag is never read. +# +# Only true no-ops qualify. A `remove` naming an organism that is on the portal # but not in Apollo really does suppress an add, so it is NOT redundant. sub _redundantOverlay { - my ($class, $portal, $live, $overlay) = @_; + my ($class, $portal, $live, $overlay, $renames, $renameTarget) = @_; my @redundant; foreach my $abbrev (sort keys %{$overlay->{add}}) { + my $note = $live->{$abbrev} ? 'already in Apollo' + : $renameTarget->{$abbrev} ? "already arriving as a rename from $renameTarget->{$abbrev}" + : !$portal->{$abbrev} ? 'not on the portal' + : undef; + next unless $note; + push @redundant, { abbrev => $abbrev, directive => 'add', reason => $overlay->{add}{$abbrev}, - note => $live->{$abbrev} ? 'already in Apollo' - : 'not on the portal', - } if $live->{$abbrev} || !$portal->{$abbrev}; + note => $note, + }; } foreach my $abbrev (sort keys %{$overlay->{remove}}) { - next if $live->{$abbrev} || $portal->{$abbrev}; + my $note = $renames->{$abbrev} + ? "being renamed to $renames->{$abbrev}, not pruned" + : (!$live->{$abbrev} && !$portal->{$abbrev}) + ? 'in neither Apollo nor the portal' + : undef; + next unless $note; + push @redundant, { abbrev => $abbrev, directive => 'remove', reason => $overlay->{remove}{$abbrev}, - note => 'in neither Apollo nor the portal', + note => $note, }; } diff --git a/Model/t/apollo.t b/Model/t/apollo.t index 3a81b362ba..65235fcf7e 100644 --- a/Model/t/apollo.t +++ b/Model/t/apollo.t @@ -9,7 +9,7 @@ my $A = 'ApiCommonModel::Model::ApolloRelease::Apollo'; my $live = $A->loadFromFile("Model/t/fixtures/apollo.json"); is(ref($live), 'HASH', 'returns a hash keyed by abbrev'); -is(scalar(keys %$live), 5, 'all five organisms parsed'); +is(scalar(keys %$live), 6, 'all six organisms parsed'); my $t = $live->{tgonME49}; is($t->{id}, 1484940, 'numeric id carried'); diff --git a/Model/t/fixtures/apollo.json b/Model/t/fixtures/apollo.json index 9cf699cbd6..d98df6b96b 100644 --- a/Model/t/fixtures/apollo.json +++ b/Model/t/fixtures/apollo.json @@ -18,5 +18,9 @@ {"id": 9000002, "commonName": "Something Renamed In The Gui", "directory": "/data/apollo_data/pfal3D7", "blatdb": "/data/apollo_data/twoBit/pfal3D7.2bit", - "annotationCount": 3, "publicMode": true} + "annotationCount": 3, "publicMode": true}, + {"id": 2458311, "commonName": "Trichoderma reesei QM6a [Nov 01, 2021]", + "directory": "/data/apollo_data/treeQM6a", + "blatdb": "/data/apollo_data/twoBit/treeQM6a.2bit", + "annotationCount": 11, "publicMode": false} ] diff --git a/Model/t/fixtures/portal.json b/Model/t/fixtures/portal.json index 561d338c81..228a98d799 100644 --- a/Model/t/fixtures/portal.json +++ b/Model/t/fixtures/portal.json @@ -823,6 +823,49 @@ "annotation_source": "GeneDB" } ] + }, + { + "name": "Trichoderma reesei QM6a", + "is_reference_strain": "1", + "is_annotated_genome": "1", + "history": [ + { + "annotation_version": "Nov 01, 2021", + "genome_version": "GCA_000167675.2", + "build_number": "56", + "public_abbrev": "treeQM6a", + "genome_source": "GenBank", + "annotation_source": "Apollo" + }, + { + "annotation_version": "Dec 13, 2012", + "genome_version": "GCA_000167675.2", + "build_number": "51", + "public_abbrev": "treeQM6a", + "genome_source": "GenBank", + "annotation_source": "GenBank" + }, + { + "annotation_source": "GenBank", + "genome_source": "GenBank", + "public_abbrev": "treeQM6a", + "build_number": "29", + "genome_version": "Mar 14, 2015", + "annotation_version": "Dec 13, 2012" + }, + { + "annotation_source": "JGI", + "genome_source": "JGI", + "build_number": "19", + "public_abbrev": "treeQM6a", + "annotation_version": "2", + "genome_version": "2" + } + ], + "name_for_filenames": "TreeseiQM6a", + "strain_abbrev": "QM6a", + "organism_abbrev": "treeQM6a", + "species_ncbi_tax_id": "51453" } ] } \ No newline at end of file diff --git a/Model/t/reconcile.t b/Model/t/reconcile.t index 886c06ef90..728ec8f287 100644 --- a/Model/t/reconcile.t +++ b/Model/t/reconcile.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 26; +use Test::More tests => 36; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; use ApiCommonModel::Model::ApolloRelease::Apollo; @@ -61,10 +61,26 @@ is($cgla->{approved}, 1, 'an overlay remove approves the prune'); my ($tgon) = grep { $_->{abbrev} eq 'tgonME49' } @{$r->{update}}; is($tgon->{public_mode}, 1, 'update carries the live publicMode through'); +# treeQM6a is the case that actually matters: hidden by curators +# (publicMode false), on the portal, reference+annotated, 11 annotations. It +# is a plain update, so a hardcoded publicMode=true would re-publish it -- +# exactly the 17-organism bug this module exists to prevent. +my ($tree) = grep { $_->{abbrev} eq 'treeQM6a' } @{$r->{update}}; +ok($tree, 'a hidden organism still qualifies for a routine update'); +is($tree->{public_mode}, 0, 'update on a HIDDEN organism carries publicMode 0, not 1'); + my ($ren) = @{$r2->{rename}}; is($ren->{public_mode}, 1, 'rename carries the live publicMode through'); is($ren->{annotation_count}, 14, 'rename carries the curation work it is protecting'); +# Same rule on the rename path. No live pair exercises it, so the rename map +# is synthetic -- but the passthrough it checks is not: a hidden organism that +# gets reclassified must not be published by the act of being renamed. +my $r5 = $R->reconcile($portal, $live, $overlay, { treeQM6a => 'tbruTREU927' }); +my ($hiddenRename) = @{$r5->{rename}}; +is($hiddenRename->{from_abbrev}, 'treeQM6a', 'the hidden organism is the one renamed'); +is($hiddenRename->{public_mode}, 0, 'rename of a HIDDEN organism carries publicMode 0, not 1'); + # --------------------------------------------------------------------------- # Attacking the invariant directly. reconcile() cannot construct a violating # result (see Reconcile.pm), so the guard is exercised at its own seam: it is a @@ -115,6 +131,20 @@ eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => 'notOnPortal' }) } like($@, qr/rename target notOnPortal is not on the portal/, 'a rename target absent from the portal dies'); +eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => '' }) }; +like($@, qr/rename target for cneoJEC21 is empty/, + 'an empty rename target dies instead of falling through as "no rename"'); + +# treeQM6a is both a target (of cneoJEC21) and a source (of tbruTREU927). +eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => 'treeQM6a', + treeQM6a => 'tbruTREU927' }) }; +like($@, qr/treeQM6a is both a rename source and a rename target/, + 'a chained rename is refused rather than applied in an arbitrary order'); + +eval { $R->reconcile($portal, $live, $overlay, { cneoJEC21 => 'tgonME49' }) }; +like($@, qr/rename target tgonME49 is already in Apollo; that is a merge, not a rename/, + 'repointing onto a live organism is refused'); + # --------------------------------------------------------------------------- # Overlay hygiene. A no-op overlay line is not an error, but it must be # reported: it is a human decision the tool did nothing with. @@ -131,3 +161,20 @@ is($redundant{tgonME49}{directive}, 'add', 'an overlay add for a live organism i is($redundant{zzzGhost}{directive}, 'remove', 'an overlay entry naming nothing at all is reported'); is($redundant{tgonME49}{reason}, 'already live, left over from last release', 'the redundant entry carries its reason so the file can be cleaned up'); + +# A rename consumes BOTH abbrevs, disarming an overlay line naming either end. +# Neither shows up in any other bucket, so this is the only place they surface. +my $renameShadowedOverlay = ApiCommonModel::Model::ApolloRelease::Overlay->parseString( + "add cdenJEC21 # approved before we knew it was a rename\n" + . "remove cneoJEC21 # thought it was gone; it was reclassified\n"); +my $r6 = $R->reconcile($portal, $live, $renameShadowedOverlay, { cneoJEC21 => 'cdenJEC21' }); +my %shadowed = map { $_->{abbrev} => $_ } @{$r6->{redundant_overlay}}; +is($shadowed{cdenJEC21}{note}, 'already arriving as a rename from cneoJEC21', + 'an overlay add naming a rename target is reported, not swallowed'); +is($shadowed{cneoJEC21}{note}, 'being renamed to cdenJEC21, not pruned', + 'an overlay remove naming a rename source is reported, not swallowed'); + +# The exclusion that must survive all of the above. +my %stillEffective = map { $_->{abbrev} => 1 } @{$r->{redundant_overlay}}; +ok(!$stillEffective{hsapREF}, + 'a remove that really does suppress an add is NOT reported as redundant'); From 69db9850a2d60a93d3cab37ad869426b85adeb14 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:22:30 -0400 Subject: [PATCH 20/42] feat(apollo): detect taxonomic renames by assembly identity Species taxon ID cannot be used: it changed in both known renames. Strain abbrev narrows candidates, identical sequence names and lengths decide, and an ambiguous match produces no rename rather than a guess. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Rename.pm | 158 +++++++++++++++++++++++++ Model/t/rename.t | 120 +++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Rename.pm create mode 100644 Model/t/rename.t diff --git a/Model/lib/perl/ApolloRelease/Rename.pm b/Model/lib/perl/ApolloRelease/Rename.pm new file mode 100644 index 0000000000..7a957ed034 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Rename.pm @@ -0,0 +1,158 @@ +package ApiCommonModel::Model::ApolloRelease::Rename; + +use strict; +use warnings; + +# Detects taxonomic renames: the same assembly appearing under a new organism +# abbrev. Species taxon ID is deliberately NOT used -- it changed in both +# known cases (cneoJEC21 5207 -> cdenJEC21 40410). +# +# Strain abbrev narrows the candidate set; identical sequence names and +# lengths decide. +# +# Getting this wrong is not a cosmetic error. Without it the tool sees the old +# abbrev vanish from the portal and the new one appear, proposes a prune plus an +# add, and orphans every annotation attached to the old Apollo organism +# (cneoJEC21 holds 14). With it the caller repoints the existing organism and +# keeps both its id and its curation work. + +# Warnings are COLLECTED, not printed -- same policy as Portal.pm. Every path +# through detect() that declines to rename is recoverable (the organism simply +# stays a prune candidate for a human to judge), and a caller treating stderr +# as proof of failure must not read that as a broken release. +my @WARNINGS; + +sub warnings { return @WARNINGS } + +# name => length. The remaining .fai columns are byte offsets into the FASTA, +# which differ between two copies of the same assembly purely because of line +# wrapping, so they carry no identity and are discarded. +# +# Returns undef -- not an empty hash -- for a file that cannot be opened, so a +# caller can tell "no index" from "index of an empty assembly". +sub readFai { + my ($class, $path) = @_; + + open(my $fh, '<', $path) or return undef; + + my %lengths; + while (my $line = <$fh>) { + chomp $line; + next unless length $line; + + my ($name, $length) = split /\t/, $line; + + # A truncated final line (samtools killed mid-write, a partial scp) yields + # a name with no length. Silently reading that as length 0 would let two + # different assemblies compare equal on their truncated tails, so refuse + # the whole file rather than trust part of it. + unless (defined $name && length $name && defined $length && $length =~ /^\d+$/) { + # Read $. before closing -- an explicit close resets it. + my $lineNumber = $.; + close $fh; + push @WARNINGS, "$path: malformed line $lineNumber; refusing to trust this index"; + return undef; + } + + $lengths{$name} = $length + 0; + } + close $fh; + + return \%lengths; +} + +sub sameAssembly { + my ($class, $pathA, $pathB) = @_; + + my $a = $class->readFai($pathA) or return 0; + my $b = $class->readFai($pathB) or return 0; + + return 0 unless scalar(keys %$a) == scalar(keys %$b); + + # An empty index matches every other empty index, which would make two + # unrelated organisms with zero-byte .fai files look like a rename. No + # assembly has zero sequences, so treat it as no evidence, not as a match. + return 0 unless scalar(keys %$a); + + foreach my $name (keys %$a) { + return 0 unless defined $b->{$name}; + return 0 unless $b->{$name} == $a->{$name}; + } + + return 1; +} + +# $orphans - arrayref of Apollo abbrevs absent from the portal +# $portal - the portal hash from Portal.pm +# $previousFai - coderef: abbrev -> path to last release's .fa.fai +# $currentFai - coderef: portal organism -> path to this build's .fai +# $strainByAbbrev - hashref: orphan abbrev -> its strain abbrev +sub detect { + my ($class, $orphans, $portal, $previousFai, $currentFai, $strainByAbbrev) = @_; + + @WARNINGS = (); + my %renames; + + foreach my $orphan (@$orphans) { + my $oldFai = $previousFai->($orphan); + + # "No previous index" is NOT "no rename". If this were silent, an organism + # whose old release directory has aged out would quietly become a prune + # candidate and take its annotations with it. Say so and let a human look. + unless ($oldFai && -e $oldFai) { + push @WARNINGS, + "$orphan: no index from the previous release at " + . ($oldFai || '(no path)') + . "; cannot test for a rename. Treating as a prune candidate."; + next; + } + + my $strain = $strainByAbbrev->{$orphan}; + + # The strain filter is an optimisation over an exhaustive scan, not a + # correctness rule -- so an orphan whose strain could not be parsed out of + # its Apollo commonName falls back to comparing against every portal + # organism. Declining instead would turn an unparseable name into a lost + # rename, and the sequence comparison is what actually decides; the filter + # only bounds how many files get opened. + unless (defined $strain) { + push @WARNINGS, + "$orphan: no strain abbrev; comparing against all " + . scalar(keys %$portal) . " portal organisms."; + } + + my @candidates = grep { + !defined($strain) || (defined($_->{strain_abbrev}) && $_->{strain_abbrev} eq $strain) + } values %$portal; + + my @matches; + foreach my $candidate (@candidates) { + next if $candidate->{abbrev} eq $orphan; + my $newFai = $currentFai->($candidate); + next unless $newFai && -e $newFai; + push @matches, $candidate->{abbrev} + if $class->sameAssembly($oldFai, $newFai); + } + + if (@matches == 1) { + $renames{$orphan} = $matches[0]; + } + elsif (@matches > 1) { + # Never guess. Picking one repoints curated annotations at the wrong + # genome -- silently, and in a way nothing downstream can detect. A + # prune candidate a human has to judge is strictly the lesser harm. + push @WARNINGS, + "$orphan: matches more than one portal organism (" + . join(', ', sort @matches) + . "); refusing to guess. Treating as a prune candidate."; + } + else { + push @WARNINGS, + "$orphan: no portal organism shares its assembly. Treating as a prune candidate."; + } + } + + return \%renames; +} + +1; diff --git a/Model/t/rename.t b/Model/t/rename.t new file mode 100644 index 0000000000..30dda77660 --- /dev/null +++ b/Model/t/rename.t @@ -0,0 +1,120 @@ +use strict; +use warnings; +use Test::More tests => 17; +use File::Temp qw(tempdir); +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Rename; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Rename'; + +my $dir = tempdir(CLEANUP => 1); + +sub writeFai { + my ($path, @lines) = @_; + open(my $fh, '>', $path) or die $!; + print $fh "$_\n" for @lines; + close $fh; +} + +# Old organism's shipped index, and a new organism with the same assembly. +writeFai("$dir/old.fai", "AE017341.1\t2300533\t60\t60\t61", "AE017342.1\t1632307\t60\t60\t61"); +writeFai("$dir/same.fai", "AE017341.1\t2300533\t99\t70\t71", "AE017342.1\t1632307\t99\t70\t71"); +writeFai("$dir/other.fai", "CP000001.1\t123456\t60\t60\t61"); + +is_deeply($R->readFai("$dir/old.fai"), + {'AE017341.1' => 2300533, 'AE017342.1' => 1632307}, + 'fai parsed to name => length, ignoring offset columns'); + +ok($R->sameAssembly("$dir/old.fai", "$dir/same.fai"), + 'identical names and lengths match despite different byte offsets'); +ok(!$R->sameAssembly("$dir/old.fai", "$dir/other.fai"), + 'different assembly does not match'); + +my $portal = { + cdenJEC21 => {abbrev => 'cdenJEC21', strain_abbrev => 'JEC21', name_for_filenames => 'CdeneoformansJEC21'}, + tgonME49 => {abbrev => 'tgonME49', strain_abbrev => 'ME49', name_for_filenames => 'TgondiiME49'}, +}; + +my $renames = $R->detect( + ['cneoJEC21'], + $portal, + sub { my ($abbrev) = @_; return "$dir/old.fai" }, + sub { my ($org) = @_; return $org->{abbrev} eq 'cdenJEC21' ? "$dir/same.fai" : "$dir/other.fai" }, + {cneoJEC21 => 'JEC21'}, +); + +is_deeply($renames, {cneoJEC21 => 'cdenJEC21'}, 'rename detected by sequence identity'); + +# A missing previous-release index must not silently mean "no rename". +my $none = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/does-not-exist.fai" }, + sub { return "$dir/same.fai" }, + {cneoJEC21 => 'JEC21'}); +is_deeply($none, {}, 'unresolvable organism yields no rename'); + +my @warnings = $R->warnings(); +like($warnings[0], qr/cneoJEC21/, 'and says so rather than staying silent'); + +# --- degenerate index files ------------------------------------------------- +# Each of these is a file that EXISTS, so the caller's -e check passes and the +# only thing standing between it and a wrong answer is readFai. + +writeFai("$dir/empty.fai"); +is_deeply($R->readFai("$dir/empty.fai"), {}, 'an empty index parses to an empty hash'); +ok(!$R->sameAssembly("$dir/empty.fai", "$dir/empty.fai"), + 'two empty indexes are NOT the same assembly -- no assembly has zero sequences'); + +# samtools killed mid-write: a final line with a name but no length. +writeFai("$dir/truncated.fai", "AE017341.1\t2300533\t60\t60\t61", "AE017342.1"); +is($R->readFai("$dir/truncated.fai"), undef, 'a truncated index is refused, not partly believed'); +ok(!$R->sameAssembly("$dir/old.fai", "$dir/truncated.fai"), + 'and so cannot match on the intact prefix of a different assembly'); +ok((grep { /truncated\.fai/ } $R->warnings()), 'a refused index is reported'); + +# --- the ambiguous case ----------------------------------------------------- +# Two portal organisms sharing a strain abbrev AND an assembly. This is the +# case where a guess would repoint curated annotations onto the wrong genome. + +my $ambiguous = { + cdenJEC21 => {abbrev => 'cdenJEC21', strain_abbrev => 'JEC21'}, + cdupJEC21 => {abbrev => 'cdupJEC21', strain_abbrev => 'JEC21'}, +}; +my $tie = $R->detect(['cneoJEC21'], $ambiguous, + sub { return "$dir/old.fai" }, + sub { return "$dir/same.fai" }, + {cneoJEC21 => 'JEC21'}); +is_deeply($tie, {}, 'two equally good matches produce NO rename rather than a guess'); +ok((grep { /cdenJEC21.*cdupJEC21|refusing to guess/ } $R->warnings()), + 'and the ambiguity is named, not swallowed'); + +# --- an orphan whose strain abbrev could not be parsed ----------------------- +# The strain filter bounds how many files get opened; sequence identity is what +# decides. So an unparseable strain must widen the search, not abandon it -- +# otherwise an Apollo commonName the parser does not understand silently costs +# a rename and orphans its annotations. + +my $noStrain = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/old.fai" }, + sub { my ($o) = @_; return $o->{abbrev} eq 'cdenJEC21' ? "$dir/same.fai" : "$dir/other.fai" }, + {}); +is_deeply($noStrain, {cneoJEC21 => 'cdenJEC21'}, + 'an unknown strain falls back to every portal organism and still finds the rename'); +ok((grep { /no strain abbrev/ } $R->warnings()), + 'the widened search is reported, since it is the slow path'); + +# A strain that matches nothing on the portal must not quietly become a +# full scan -- it is a real filter, and an empty candidate set is an answer. +my $wrongStrain = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/old.fai" }, + sub { return "$dir/same.fai" }, + {cneoJEC21 => 'NOSUCHSTRAIN'}); +is_deeply($wrongStrain, {}, 'a strain matching no portal organism yields no rename'); + +# The current index may legitimately not exist yet for some portal organism; +# that must skip the candidate, not abort the whole orphan. +my $missingCurrent = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/old.fai" }, + sub { my ($o) = @_; return $o->{abbrev} eq 'cdenJEC21' ? "$dir/same.fai" : "$dir/gone.fai" }, + {}); +is_deeply($missingCurrent, {cneoJEC21 => 'cdenJEC21'}, + 'a candidate with no current index is skipped, not fatal'); From d55a18a2a246de8d1b07baba0445b611d34a691f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:29:13 -0400 Subject: [PATCH 21/42] feat(apollo): add URL absolutization with a mandatory post-condition check A text pass rather than a typed transformation: ~25 of these URLs live inside HTML blobs and JavaScript function bodies where a typed getApolloObject cannot reach. The previous script did the same rewrite with no verification, so a missed URL became a silently empty track. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Absolutize.pm | 86 ++++++++++++++ Model/t/absolutize.t | 132 +++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Absolutize.pm create mode 100644 Model/t/absolutize.t diff --git a/Model/lib/perl/ApolloRelease/Absolutize.pm b/Model/lib/perl/ApolloRelease/Absolutize.pm new file mode 100644 index 0000000000..b5bf80de07 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Absolutize.pm @@ -0,0 +1,86 @@ +package ApiCommonModel::Model::ApolloRelease::Absolutize; + +use strict; +use warnings; + +# Apollo embeds JBrowse and reads its config off disk, so every URL in that +# config must be absolute. The jbrowse* scripts emit site-relative "/a/..." +# because they normally serve a website. +# +# This is a text pass rather than a typed transformation because roughly 25 of +# these live inside free text -- HTML blobs, onClick and menuTemplate URLs, and +# JavaScript function bodies -- where a typed getApolloObject() cannot reach. +# (The applicationType=apollo code path that was supposed to handle this is +# unimplemented: Store::makeUrlTemplate is a `die "TODO"`, and 39 track classes +# implement getJBrowseObject against exactly 1 implementing getApolloObject.) +# +# The post-condition check is not optional. The previous script did the same +# rewrite with no verification, so a missed URL became a track that silently +# 404s inside Apollo: the config loads, the track appears, and it is empty. + +# A site-root URL is "/a/" that is NOT preceded by an ordinary path or word +# character. Written as a NEGATIVE lookbehind rather than a positive one so +# that a "/a/" at the very start of the string also matches -- a positive +# lookbehind has nothing to match there and silently misses it. +# +# The class was not guessed; it is the measured set. Across the generated +# config ($GUS_HOME/lib/jbrowse/{functions,jbrowse,jbrowse_embed, +# apollo_gene_tracks,tracks}.conf and auto_generated/*/tracks.conf) the only +# characters that ever precede "/a/" are: = (91), ' (14), " (13). Across +# Model/lib/perl the set is: " (27), ' (7), = (1), and one `l`. +# +# That `l` is the case this pattern exists to EXCLUDE -- "$projectUrl/a/service/..." +# already carries an absolute base, and prefixing it again would produce +# "$projectUrlhttps://...". Any word or path character before "/a/" means the +# "/a/" is a continuation, not a site root. +# +# "//a/" is therefore also left alone: it is protocol-relative, already carries +# a host position, and prefixing it would yield "//https://...". Zero occur in +# the real config; the behaviour is pinned by test rather than left to accident. +my $RELATIVE = qr{(? $shown ? " (first $shown shown)" : "") . ":\n" + . join('', map { " $_\n" } @found); +} + +1; diff --git a/Model/t/absolutize.t b/Model/t/absolutize.t new file mode 100644 index 0000000000..fce600c22f --- /dev/null +++ b/Model/t/absolutize.t @@ -0,0 +1,132 @@ +use strict; +use warnings; +use Test::More; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Absolutize; + +my $A = 'ApiCommonModel::Model::ApolloRelease::Absolutize'; +my $BASE = 'https://veupathdb.org'; + +is($A->rewrite('"urlTemplate":"/a/service/jbrowse/store?data=x"', $BASE), + '"urlTemplate":"https://veupathdb.org/a/service/jbrowse/store?data=x"', + 'store URL in JSON'); + +is($A->rewrite("", $BASE), + "", + 'URL inside an HTML blob'); + +is($A->rewrite("function(t,f) { return '/a/app/record/gene/' + f.get('name') }", $BASE), + "function(t,f) { return 'https://veupathdb.org/a/app/record/gene/' + f.get('name') }", + 'URL inside a JavaScript function body'); + +is($A->rewrite('"baseUrl":"/a/service/jbrowse"', $BASE), + '"baseUrl":"https://veupathdb.org/a/service/jbrowse"', + 'bare service base with no trailing path'); + +is($A->rewrite('already https://veupathdb.org/a/x', $BASE), + 'already https://veupathdb.org/a/x', + 'an already-absolute URL is not rewritten twice'); + +is($A->rewrite('a path like /data/a/thing', $BASE), + 'a path like /data/a/thing', + 'a mid-path /a/ that is not a site-root URL is left alone'); + +eval { $A->assertNoRelative('{"url":"/a/service/x"}', 'trackList.json') }; +like($@, qr/trackList\.json/, 'the assertion names the offending file'); + +# --------------------------------------------------------------------------- +# Every character MEASURED to precede "/a/" in the real generated config and in +# Model/lib/perl. Measured frequencies: +# generated config: = 91, ' 14, " 13 +# Model/lib/perl: " 27, ' 7, = 1, l 1 +# The lookbehind was validated against this set rather than assumed. +# --------------------------------------------------------------------------- + +is($A->rewrite('"url":"?data=/a/service/jbrowse/tracks/tgonME49"', $BASE), + '"url":"?data=https://veupathdb.org/a/service/jbrowse/tracks/tgonME49"', + 'preceded by "=" -- the single most common case (91 of 118) in real config'); + +is($A->rewrite(q{href='/a/app/record/gene/X'}, $BASE), + q{href='https://veupathdb.org/a/app/record/gene/X'}, + 'preceded by a single quote'); + +is($A->rewrite('baseUrl => "/a/service/jbrowse"', $BASE), + 'baseUrl => "https://veupathdb.org/a/service/jbrowse"', + 'preceded by a double quote'); + +# The one `l` measured in Model/lib/perl is "$projectUrl/a/service/..." -- a +# string that already carries an absolute base. Rewriting it would produce +# "$projectUrlhttps://...". This is the case the lookbehind exists to reject. +is($A->rewrite('my $u = "$projectUrl/a/service/jbrowse/store?data=x";', $BASE), + 'my $u = "$projectUrl/a/service/jbrowse/store?data=x";', + 'an interpolated absolute base already precedes it -- left alone'); + +# Not measured in the config, but in the character class, so pin the intent. +is($A->rewrite("/a/service/x", $BASE), + 'https://veupathdb.org/a/service/x', + 'at the very start of the string -- a positive lookbehind would miss this'); + +is($A->rewrite("line one\n/a/service/x", $BASE), + "line one\nhttps://veupathdb.org/a/service/x", + 'at the start of a line inside a multi-line blob'); + +is($A->rewrite('(/a/images/x.png)', $BASE), + '(https://veupathdb.org/a/images/x.png)', + 'preceded by an open paren'); + +# --------------------------------------------------------------------------- +# Step 6 questions +# --------------------------------------------------------------------------- + +# rewrite() trims a trailing slash from $base. @_ is aliased in Perl, so the +# worry is real -- but `my (...) = @_` copies, so the caller's variable is not +# touched. Cheap to pin; expensive to discover later. +my $caller_base = 'https://veupathdb.org/'; +$A->rewrite('"/a/x"', $caller_base); +is($caller_base, 'https://veupathdb.org/', + "rewrite does not mutate the caller's base variable through \@_ aliasing"); + +is($A->rewrite('"/a/x"', 'https://veupathdb.org///'), '"https://veupathdb.org/a/x"', + 'trailing slashes on the base are trimmed, not doubled'); + +# Idempotence: after one pass the "/a/" is preceded by the base's last +# character, which the lookbehind rejects. Assert it rather than reason it. +my $once = $A->rewrite('"baseUrl":"/a/service/jbrowse","img":"/a/images/x.png"', $BASE); +my $twice = $A->rewrite($once, $BASE); +is($twice, $once, 'rewrite is idempotent -- a second pass does not double-prefix'); + +# A URL absolute to a DIFFERENT host is left alone. This is right, not an +# oversight: the real config contains "https://apollo.veupathdb.org/annotator/..." +# and "https://www.ncbi.nlm.nih.gov/...", which are deliberate off-site links. +# assertNoRelative is about RELATIVE URLs, so it must not flag them either. +is($A->rewrite('"u":"https://other.org/a/x"', $BASE), '"u":"https://other.org/a/x"', + 'a URL absolute to a different host is left alone'); +ok($A->assertNoRelative('"u":"https://other.org/a/x"', 'tracks.conf'), + 'assertNoRelative does not flag a foreign absolute host'); + +# Protocol-relative "//a/" already occupies the host position; prefixing it +# would yield "//https://...". Zero occur in the real config -- the behaviour +# is pinned by test so it stays a decision rather than an accident. +is($A->rewrite('"u":"//a/service/x"', $BASE), '"u":"//a/service/x"', + 'a protocol-relative //a/ is left alone'); + +# The cap on shown findings is a summary, not a truncation: the COUNT is what +# distinguishes one missed edge case from a file the rewrite never touched. +my $many = join(' ', map { qq{"/a/service/$_"} } 1 .. 10); +eval { $A->assertNoRelative($many, 'tracks.conf') }; +like($@, qr/10 site-relative URL/, 'the assertion reports the total count, not just the samples'); +like($@, qr/first 3 shown/, 'and says the listing is a sample'); +my @lines = grep { /^ / } split /\n/, $@; +is(scalar @lines, 3, 'exactly three findings are listed'); + +ok($A->assertNoRelative($A->rewrite($many, $BASE), 'tracks.conf'), + 'assertNoRelative passes on text that rewrite has processed'); + +ok($A->assertNoRelative(undef, 'tracks.conf'), 'undef text is vacuously clean'); +is($A->rewrite(undef, $BASE), undef, 'undef text rewrites to undef'); + +# An empty base would leave every URL still relative while reporting success. +eval { $A->rewrite('"/a/x"', '') }; +like($@, qr/non-empty base/, 'an empty base is refused rather than silently no-op'); + +done_testing(); From 242ee5e12eb3d9a0e28f0c4a2576bd4d263e2bc0 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:34:11 -0400 Subject: [PATCH 22/42] fix(apollo): report unreadable indexes instead of reading them as a changed genome Three review findings: - readFai warned only on a malformed line, not on an unopenable file. A permissions problem or a missing directory degraded silently to "no match", which sameAssembly cannot distinguish from a different assembly. - The ambiguity test alternated "names both candidates" with "says it is refusing", so a generic refusal naming neither would have kept it green. Split into two assertions and confirmed by mutation. - A candidate skipped for a missing current index fell through to "no portal organism shares its assembly", which is false and invites approving a prune over a build problem. Checked and uncheckable candidates are now counted and reported separately. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Rename.pm | 45 ++++++++++++++++++-- Model/t/rename.t | 59 ++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Rename.pm b/Model/lib/perl/ApolloRelease/Rename.pm index 7a957ed034..0b734d9ab4 100644 --- a/Model/lib/perl/ApolloRelease/Rename.pm +++ b/Model/lib/perl/ApolloRelease/Rename.pm @@ -33,7 +33,15 @@ sub warnings { return @WARNINGS } sub readFai { my ($class, $path) = @_; - open(my $fh, '<', $path) or return undef; + # An unopenable index -- bad permissions, a missing parent directory, a dead + # symlink -- must not degrade silently. sameAssembly() turns this undef into + # a plain 0, indistinguishable from "a genuinely different assembly", so + # without this warning an environment problem reads to the operator as "the + # genome really did change" and gets a prune approved. + open(my $fh, '<', $path) or do { + push @WARNINGS, "$path: cannot be read ($!); treating it as no evidence of a rename"; + return undef; + }; my %lengths; while (my $line = <$fh>) { @@ -125,11 +133,23 @@ sub detect { !defined($strain) || (defined($_->{strain_abbrev}) && $_->{strain_abbrev} eq $strain) } values %$portal; + # Counted, not just skipped. "I compared it and it differs" and "I never + # got to compare it" are opposite conclusions for the operator, and below + # they must not produce the same sentence. my @matches; + my $checked = 0; + my $uncheckable = 0; + foreach my $candidate (@candidates) { next if $candidate->{abbrev} eq $orphan; + my $newFai = $currentFai->($candidate); - next unless $newFai && -e $newFai; + unless ($newFai && -e $newFai) { + $uncheckable++; + next; + } + + $checked++; push @matches, $candidate->{abbrev} if $class->sameAssembly($oldFai, $newFai); } @@ -146,9 +166,28 @@ sub detect { . join(', ', sort @matches) . "); refusing to guess. Treating as a prune candidate."; } + # Nothing matched -- but WHY nothing matched decides what the operator does + # next. "Its assembly is gone from the portal" invites approving a prune. + # "I could not open the indexes" invites fixing the build. Saying the + # first when the second is true is how curated annotations get discarded + # over a missing file, so the two cases get different sentences. + elsif ($checked == 0 && $uncheckable) { + push @WARNINGS, + "$orphan: none of its $uncheckable candidate portal organisms could be checked " + . "(no current index for any of them). This is NOT evidence the genome changed. " + . "Treating as a prune candidate."; + } + elsif ($checked == 0) { + push @WARNINGS, + "$orphan: no portal organism was even a candidate" + . (defined $strain ? " for strain '$strain'" : '') + . ". Treating as a prune candidate."; + } else { push @WARNINGS, - "$orphan: no portal organism shares its assembly. Treating as a prune candidate."; + "$orphan: none of the $checked portal organisms checked shares its assembly" + . ($uncheckable ? " ($uncheckable more had no current index and were skipped)" : '') + . ". Treating as a prune candidate."; } } diff --git a/Model/t/rename.t b/Model/t/rename.t index 30dda77660..3086486bdd 100644 --- a/Model/t/rename.t +++ b/Model/t/rename.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 17; +use Test::More tests => 27; use File::Temp qw(tempdir); use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Rename; @@ -84,8 +84,14 @@ my $tie = $R->detect(['cneoJEC21'], $ambiguous, sub { return "$dir/same.fai" }, {cneoJEC21 => 'JEC21'}); is_deeply($tie, {}, 'two equally good matches produce NO rename rather than a guess'); -ok((grep { /cdenJEC21.*cdupJEC21|refusing to guess/ } $R->warnings()), - 'and the ambiguity is named, not swallowed'); +# Two separate assertions on purpose. Alternated into one `ok`, a future +# generic "ambiguous match; refusing to guess" that named neither candidate +# would keep the test green -- and naming both is the entire point, since the +# operator has to go and look at both genomes to break the tie. +my @tieWarnings = $R->warnings(); +ok((grep { /refus/i } @tieWarnings), 'the refusal to guess is stated'); +ok((grep { /cdenJEC21/ && /cdupJEC21/ } @tieWarnings), + 'and BOTH candidate abbrevs are named, so the operator knows what to compare'); # --- an orphan whose strain abbrev could not be parsed ----------------------- # The strain filter bounds how many files get opened; sequence identity is what @@ -118,3 +124,50 @@ my $missingCurrent = $R->detect(['cneoJEC21'], $portal, {}); is_deeply($missingCurrent, {cneoJEC21 => 'cdenJEC21'}, 'a candidate with no current index is skipped, not fatal'); + +# --- an index that exists (or is named) but cannot be opened ----------------- +# sameAssembly collapses this to 0, which is the same answer as "a different +# assembly". The collapse is fail-safe, but it must never be SILENT: an +# operator told "the assembly differs" approves a prune, where "I could not +# read the file" sends them to fix the build. + +my $noSuchDir = "$dir/no-such-dir/genome.fasta.fai"; +is($R->readFai($noSuchDir), undef, 'an index under a missing directory returns undef'); +ok((grep { /no-such-dir/ } $R->warnings()), + 'and the unreadable path is named, not silently treated as a different assembly'); + +SKIP: { + my $locked = "$dir/locked.fai"; + writeFai($locked, "AE017341.1\t2300533\t60\t60\t61"); + chmod 0000, $locked; + skip 'running with rights that ignore file modes (root?)', 2 if -r $locked; + + is($R->readFai($locked), undef, 'an unreadable index returns undef'); + ok((grep { /locked\.fai/ && /Permission denied/ } $R->warnings()), + 'and the warning names the path and the reason'); +} + +# --- "could not check" must not be reported as "assembly differs" ------------ +# Every candidate skipped for a missing current index used to fall through to +# "no portal organism shares its assembly", which is false and points the +# operator at exactly the wrong conclusion. + +my $unreadableAll = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/old.fai" }, + sub { return "$dir/gone.fai" }, + {}); +is_deeply($unreadableAll, {}, 'no rename when nothing could be checked'); +my @uncheckable = $R->warnings(); +ok((grep { /could be checked|NOT evidence/ } @uncheckable), + 'the warning says the candidates could not be checked'); +ok(!(grep { /shares its assembly/ } @uncheckable), + 'and does NOT claim the assembly was compared and differs'); + +# The genuinely-differs case must still say so, and must still be distinct. +my $reallyDiffers = $R->detect(['cneoJEC21'], $portal, + sub { return "$dir/old.fai" }, + sub { return "$dir/other.fai" }, + {}); +is_deeply($reallyDiffers, {}, 'no rename when the assemblies genuinely differ'); +ok((grep { /shares its assembly/ } $R->warnings()), + 'and that case is reported as a real comparison, not as an unreadable one'); From cb80d4464bb761fb52f676ee6917bc166911b90f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:44:07 -0400 Subject: [PATCH 23/42] polish(apollo): tighten base validation, a loose assertion, and exception shape Absolutize now validates the base so its idempotency guarantee is true rather than approximately true. Rename test 6 required the abbrev but not the reason. Reconcile exception entries now carry public_mode for symmetry with update and rename. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Absolutize.pm | 40 ++++++++++++- Model/lib/perl/ApolloRelease/Reconcile.pm | 7 +++ Model/t/absolutize.t | 65 ++++++++++++++++++++++ Model/t/reconcile.t | 16 +++++- Model/t/rename.t | 12 +++- 5 files changed, 135 insertions(+), 5 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Absolutize.pm b/Model/lib/perl/ApolloRelease/Absolutize.pm index b5bf80de07..3f2c6b9a1a 100644 --- a/Model/lib/perl/ApolloRelease/Absolutize.pm +++ b/Model/lib/perl/ApolloRelease/Absolutize.pm @@ -17,7 +17,29 @@ use warnings; # The post-condition check is not optional. The previous script did the same # rewrite with no verification, so a missed URL became a track that silently # 404s inside Apollo: the config loads, the track appears, and it is empty. - +# +# This module is deliberately IO-FREE: it takes and returns strings, and the +# caller owns reading and writing the files. That is what lets the whole of it +# be unit tested with no fixtures and no temp directories. Do not add a +# rewriteFile()/rewriteDir() convenience here -- it would pull disk coupling +# back into every test that touches this logic. + +# --------------------------------------------------------------------------- +# SHARED PATTERN -- rewrite() and assertNoRelative() MUST use this exact qr//. +# +# assertNoRelative is the post-condition for rewrite: it validates precisely +# what rewrite changes, no more and no less. That equivalence is what makes +# the check meaningful, and it holds only because both read the same $RELATIVE. +# Tightening or loosening one definition of "relative" without the other breaks +# it silently and in whichever direction you chose: +# - a pattern rewrite matches but the assertion does not -> URLs get +# prefixed with nothing verifying them; +# - a pattern the assertion matches but rewrite does not -> every release +# dies on a "relative" URL that no rewrite could ever remove. +# Neither shows up as a test failure unless you also change the tests. If the +# definition of a site-root URL really must change, change it HERE, once. +# --------------------------------------------------------------------------- +# # A site-root URL is "/a/" that is NOT preceded by an ordinary path or word # character. Written as a NEGATIVE lookbehind rather than a positive one so # that a "/a/" at the very start of the string also matches -- a positive @@ -51,13 +73,27 @@ sub rewrite { # never reaches through to the caller's variable. Pinned by test. $base =~ s{/+$}{}; + # Idempotency is a GUARANTEE, not a hope -- and it is only a guarantee for a + # base that cannot itself end in a delimiter the lookbehind treats as a site + # root. A base ending in one of [ ' " ( = \s ] would make its own last + # character re-trigger the match on a second pass: + # rewrite('"/a/x"', "https://foo.org'") -> "https://foo.org'/a/x" + # rewrite(that, same base) -> "https://foo.org'https://foo.org'/a/x" + # No real base looks like that, which is exactly why nothing would catch it. + # So constrain the base to a plain absolute http(s) URL with no delimiter + # characters ANYWHERE in it, and the comment below is then true by + # construction rather than true by luck. + die "Absolutize::rewrite requires a plain absolute http(s) base URL, got '$base'\n" + unless $base =~ m{^https?://[^\s'"(=]+$}; + $text =~ s{$RELATIVE}{$base/a/}g; return $text; } # Idempotent by construction: after a rewrite the "/a/" is preceded by the last -# character of the base (a host character), which the lookbehind rejects. +# character of the base, which rewrite() has validated cannot be a delimiter, +# and which the lookbehind therefore rejects. sub assertNoRelative { my ($class, $text, $label) = @_; diff --git a/Model/lib/perl/ApolloRelease/Reconcile.pm b/Model/lib/perl/ApolloRelease/Reconcile.pm index 2387e704af..69d79ab23e 100644 --- a/Model/lib/perl/ApolloRelease/Reconcile.pm +++ b/Model/lib/perl/ApolloRelease/Reconcile.pm @@ -79,6 +79,13 @@ sub reconcile { annotation_count => $apollo->{annotation_count}, is_reference => $organism->{is_reference}, is_annotated => $organism->{is_annotated}, + # An exception stays in Apollo untouched, so its CURRENT visibility is + # part of what a reader needs alongside is_reference/is_annotated to + # judge it. Report-only: no command is generated from an exception, so + # unlike `update` and `rename` this value is never echoed back to + # Apollo. It is carried for symmetry with them, and so a report does + # not have to reach back into $live for one field. + public_mode => $apollo->{public_mode}, }; next; } diff --git a/Model/t/absolutize.t b/Model/t/absolutize.t index fce600c22f..372e44071e 100644 --- a/Model/t/absolutize.t +++ b/Model/t/absolutize.t @@ -129,4 +129,69 @@ is($A->rewrite(undef, $BASE), undef, 'undef text rewrites to undef'); eval { $A->rewrite('"/a/x"', '') }; like($@, qr/non-empty base/, 'an empty base is refused rather than silently no-op'); +# --------------------------------------------------------------------------- +# The idempotency guarantee holds only for a base that cannot itself end in a +# delimiter the lookbehind treats as a site root. rewrite() enforces that +# rather than assuming it, so the comment in the module is true by +# construction. The counterexample that motivated the check: +# rewrite('"/a/x"', "https://foo.org'") -> "https://foo.org'/a/x" +# and a second pass then double-prefixes, because the base's own trailing +# quote satisfies the lookbehind. +# --------------------------------------------------------------------------- +for my $bad ("https://foo.org'", 'https://foo.org"', 'https://foo.org(', + 'https://foo.org=', 'https://foo.org ') { + eval { $A->rewrite('"/a/x"', $bad) }; + like($@, qr/plain absolute http\(s\) base URL/, + "a base ending in a delimiter is refused: '$bad'"); +} + +# Not a URL at all, and a scheme we do not serve: both would produce nonsense +# that still looks like a successful rewrite. +for my $bad ('veupathdb.org', '/veupathdb', 'ftp://veupathdb.org') { + eval { $A->rewrite('"/a/x"', $bad) }; + like($@, qr/plain absolute http\(s\) base URL/, + "a base that is not an absolute http(s) URL is refused: '$bad'"); +} + +# ...and for every character class the validated base MAY end in, a second pass +# is a no-op. This is the guarantee itself, exercised rather than reasoned. +for my $good ('https://veupathdb.org', # letter + 'http://veupathdb.org', # the other permitted scheme + 'https://veupathdb.org:8080', # digit + 'https://beta-w1.veupathdb.org',# hyphen (interior) / letter + 'https://veupathdb.org/a_b', # underscore + 'https://veupathdb.org/x~', # tilde + 'https://veupathdb.org/x.', # dot + 'https://veupathdb.org/x-', # hyphen + ) { + my $one = $A->rewrite('"baseUrl":"/a/service","img":"/a/images/x.png"', $good); + is($A->rewrite($one, $good), $one, "idempotent for a base ending as in '$good'"); + ok($A->assertNoRelative($one, 'tracks.conf'), + "and one pass satisfies the post-condition for '$good'"); +} + +# A PARTIALLY rewritten string -- some URLs already absolute, some still +# relative -- is the realistic shape of a half-failed rewrite, and the count is +# what diagnoses it. The existing count test uses ten identical relatives, +# which cannot show that the absolute ones are excluded from the total. +my $partial = join(' ', + qq{"a":"$BASE/a/service/1"}, + qq{"b":"/a/service/2"}, + qq{"c":"$BASE/a/images/3.png"}, + qq{"d":"/a/service/4"}, + qq{"e":"https://other.org/a/x"}, + qq{"f":"//a/service/5"}, + qq{"g":"/data/a/thing"}, + qq{"h":"/a/service/6"}, +); +eval { $A->assertNoRelative($partial, 'tracks.conf') }; +like($@, qr/3 site-relative URL/, + 'a partially rewritten string counts only the STILL-relative URLs'); +unlike($@, qr/first \d+ shown/, + 'and lists them all when there are no more than three'); +ok($A->assertNoRelative($A->rewrite($partial, $BASE), 'tracks.conf'), + 'rewriting the partially rewritten string finishes the job'); +is($A->rewrite($partial, $BASE), $A->rewrite($A->rewrite($partial, $BASE), $BASE), + 'and re-rewriting a mixed string is still idempotent'); + done_testing(); diff --git a/Model/t/reconcile.t b/Model/t/reconcile.t index 728ec8f287..2725f13ade 100644 --- a/Model/t/reconcile.t +++ b/Model/t/reconcile.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 36; +use Test::More tests => 38; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; use ApiCommonModel::Model::ApolloRelease::Apollo; @@ -81,6 +81,20 @@ my ($hiddenRename) = @{$r5->{rename}}; is($hiddenRename->{from_abbrev}, 'treeQM6a', 'the hidden organism is the one renamed'); is($hiddenRename->{public_mode}, 0, 'rename of a HIDDEN organism carries publicMode 0, not 1'); +# An exception generates no command, so this is report-only -- but "stays in +# Apollo unchanged" is only readable if the report can show WHAT it stays as. +# Visibility belongs next to is_reference/is_annotated for that judgement. +my ($exc) = grep { $_->{abbrev} eq 'tbruLister427_2018' } @{$r->{exception}}; +is($exc->{public_mode}, 1, 'exception carries the live publicMode through'); + +# ...and it is the LIVE value, not a constant that happens to match. No hidden +# organism is a real exception in the fixture, so hide one synthetically. +my %hiddenLive = %$live; +$hiddenLive{tbruLister427_2018} = { %{$live->{tbruLister427_2018}}, public_mode => 0 }; +my $rHiddenExc = $R->reconcile($portal, \%hiddenLive, $overlay, {}); +my ($hiddenExc) = grep { $_->{abbrev} eq 'tbruLister427_2018' } @{$rHiddenExc->{exception}}; +is($hiddenExc->{public_mode}, 0, 'exception on a HIDDEN organism carries publicMode 0, not 1'); + # --------------------------------------------------------------------------- # Attacking the invariant directly. reconcile() cannot construct a violating # result (see Reconcile.pm), so the guard is exercised at its own seam: it is a diff --git a/Model/t/rename.t b/Model/t/rename.t index 3086486bdd..2681904336 100644 --- a/Model/t/rename.t +++ b/Model/t/rename.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 27; +use Test::More tests => 28; use File::Temp qw(tempdir); use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Rename; @@ -53,7 +53,15 @@ my $none = $R->detect(['cneoJEC21'], $portal, is_deeply($none, {}, 'unresolvable organism yields no rename'); my @warnings = $R->warnings(); -like($warnings[0], qr/cneoJEC21/, 'and says so rather than staying silent'); +# Naming the organism is not "saying so": a warning that printed the abbrev and +# nothing else would pass a /cneoJEC21/ match while telling the operator nothing +# actionable. Require the REASON too -- that no previous-release index was +# found -- since that is what distinguishes an aged-out release directory from +# a genuine "this organism is gone". +like($warnings[0], qr/cneoJEC21/, + 'the warning names the organism it could not resolve'); +like($warnings[0], qr/no index from the previous release/, + 'and gives the reason -- no previous-release index -- not just the name'); # --- degenerate index files ------------------------------------------------- # Each of these is a file that EXISTS, so the caller's -e check passes and the From 69e4c9860efd660f5602da6df9da980cd12bbf6b Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:45:24 -0400 Subject: [PATCH 24/42] Apollo spec: record the measured jbrowse script baseline, and two defects Task 9 measured every producing script against three organisms on build 71. The five track producers are sound -- clean exits, zero stderr, plausible counts across two component databases including partitioned pfal3D7. Build on them. Two defects found: 16. jbrowseRefSeqs is dead for every organism. e0e9a61bb commented out the getCacheFile/setCacheFile accessors but left two callers. Served through responseFromCommand, so it corrupts the response body rather than just logging. 17. The two script families disagree about which organism abbrev they take. apidb.organism holds abbrev=cneoJEC21, public_abbrev= cdenJEC21; jbrowseTracks consumes the public one, the five track producers consume the internal one. Also measured: no organism carries a [tracks.refseq] stanza, so the planned strip step currently has nothing to strip. Findings salvaged from a stub Task 9 created in the wrong repo. Co-Authored-By: Claude Opus 5 --- ...026-08-21-apollo-release-package-design.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 5d59e49d8c..75cc583246 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -44,6 +44,8 @@ Every row measured on 2026-08-21, not inferred. | 13 | The renamed genome is byte-identical | b68 `cneoJEC21.fa.fai` vs b71 `CdeneoformansJEC21/genome.fasta.fai`: 14 sequences, identical names and lengths. | | 14 | `applicationType=apollo` is unimplemented | `Store::makeUrlTemplate` is `die "TODO: make apollo work"`. `getConfigurationObject` dispatches `apollo` to the abstract `getApolloObject`; **39** classes implement `getJBrowseObject`, **1** implements `getApolloObject`. | | 15 | 17 live organisms are deliberately hidden, and the update command would un-hide them | `publicMode=false` on 17 organisms; 16 are on the portal and 15 qualify as reference+annotated, so they are curator-hidden, not retired. 3 carry annotations (`iscaPalLabHiFi` 20, `treeQM6a` 11, `etenHoughton2021` 2). Paul's `updateOrganismInfo` curl hardcodes `"publicMode":"true"`, so running the generated commands re-publishes all 17. | +| 16 | `jbrowseRefSeqs` is dead for every organism | Exit 255, 211 bytes on stderr: `Can't locate object method "getCacheFile"`. Commit `e0e9a61bb` commented out the `getCacheFile`/`setCacheFile` accessors in `JBrowseUtil.pm` but left `printFromCache()` and `setCacheFileName()` calling them. Organism-independent; dies before any DB work. Served through `responseFromCommand`, so it corrupts the response body, not just a log. | +| 17 | The two script families disagree about which organism abbrev they take | `apidb.organism` holds `abbrev = cneoJEC21`, `public_abbrev = cdenJEC21`. `jbrowseTracks` consumes the **public** abbrev and returns the internal one; the five track producers consume the **internal** abbrev and key `auto_generated//` off it. Feeding the wrong one gives exit 2, or a wrong-organism answer. | ### Set sizes as of build 71 @@ -319,6 +321,90 @@ differ only by known, enumerable transformations. So: Probe organisms: `tgonME49` (rich track set, 10 annotations in Apollo), `pfal3D7` (different component DB, partitioned queries), and `cdenJEC21` (the rename). +### Measured baseline of the `jbrowse*` producing scripts (2026-08-21) + +Measured on cedar against build **71**, model **UniDB**, webServices +`/var/www/Common/apiSiteFilesMirror/webServices`, +`GUS_HOME=/var/www/EuPathDB/eupathdb.jbrestel/gus_home`. +Every script was run directly; nothing was built (`bld`/`wb` deliberately not run). + +### Results + +| script | organism | exit | stdout B | stderr B | JSON | tracks | +|---|---|---|---|---|---|---| +| jbrowseRnaAndChipSeqTracks RNASeq | tgonME49 | 0 | 935200 | 0 | yes | 474 | +| jbrowseRnaAndChipSeqTracks ChIPSeq | tgonME49 | 0 | 30852 | 0 | yes | 21 | +| jbrowseRNASeqJunctionTracks | tgonME49 | 0 | 18338 | 0 | yes | 4 | +| jbrowseOrganismSpecificTracks | tgonME49 | 0 | 146486 | 0 | yes | 158 | +| jbrowseDNASeqTracks | tgonME49 | 0 | 310296 | 0 | yes | 192 | +| jbrowseRefSeqs | tgonME49 | **255** | 0 | **211** | no | — | +| jbrowseTracks geneAnnotationTracks | tgonME49 | 0 | 352 | 0 | yes | 0 (by design) | +| jbrowseRnaAndChipSeqTracks RNASeq | pfal3D7 | 0 | 1899853 | 0 | yes | 880 | +| jbrowseRnaAndChipSeqTracks ChIPSeq | pfal3D7 | 0 | 355151 | 0 | yes | 230 | +| jbrowseRNASeqJunctionTracks | pfal3D7 | 0 | 18350 | 0 | yes | 4 | +| jbrowseOrganismSpecificTracks | pfal3D7 | 0 | 132196 | 0 | yes | 103 | +| jbrowseDNASeqTracks | pfal3D7 | 0 | 2513500 | 0 | yes | 1611 | +| jbrowseRefSeqs | pfal3D7 | **255** | 0 | **211** | no | — | +| jbrowseTracks geneAnnotationTracks | pfal3D7 | 0 | 351 | 0 | yes | 0 (by design) | +| all five track scripts | **cdenJEC21** | **2** | 0 | **271** | no | — | +| jbrowseTracks geneAnnotationTracks | cdenJEC21 | 0 | 363 | 0 | yes | 0 (by design) | +| jbrowseRnaAndChipSeqTracks RNASeq | cneoJEC21 | 0 | 417374 | 0 | yes | 198 | +| jbrowseRnaAndChipSeqTracks ChIPSeq | cneoJEC21 | 0 | 13 | 0 | yes | **0** | +| jbrowseRNASeqJunctionTracks | cneoJEC21 | 0 | 18366 | 0 | yes | 4 | +| jbrowseOrganismSpecificTracks | cneoJEC21 | 0 | 168283 | 0 | yes | 15 | +| jbrowseDNASeqTracks | cneoJEC21 | 0 | 13 | 0 | yes | **0** | +| jbrowseRefSeqs | cneoJEC21 | **255** | 0 | **211** | no | — | + +### Static config + +| file | tgonME49 | pfal3D7 | cneoJEC21 | cdenJEC21 | +|---|---|---|---|---| +| `auto_generated//` | present | present | present | **absent** | +| `tracks.conf` bytes | 34532 | 29469 | 457 | — | +| `[track…]` stanzas | 34 | 29 | 1 (`gcContent`) | — | +| `[tracks.refseq]` stanza | 0 | 0 | 0 | — | + +`$GUS_HOME/lib/jbrowse/functions.conf` exists, 106221 bytes, non-empty. +No organism carries a `[tracks.refseq]` stanza, so the "strip refseq" step the tool +was expected to perform currently has nothing to strip — do not assume it is dead code +without re-checking a wider organism sample. + +### Defects found + +1. **`jbrowseRefSeqs` is dead for every organism.** Exit 255, empty stdout, 211 bytes on + stderr: + `Can't locate object method "getCacheFile" via package "ApiCommonModel::Model::JBrowseUtil" at .../JBrowseUtil.pm line 419`. + Commit `e0e9a61bb` ("comment out stuff to do with CACHE") commented out the + `getCacheFile`/`setCacheFile` accessors (JBrowseUtil.pm lines 34–35) but left both + `printFromCache()` (line 419) and `setCacheFileName()` calling them. The die happens + before any DB work, so it is organism-independent. Because these scripts are also + served through `responseFromCommand`, which merges stderr into the JSON body, this is + a payload-corruption bug, not just a log line. + +2. **Organism-abbrev namespace mismatch.** `apidb.organism` for this genome is + `abbrev = cneoJEC21`, `public_abbrev = cdenJEC21`. `jbrowseTracks` takes the **public** + abbrev (its SQL selects `where public_abbrev = ?`) and returns the internal one; the + five track-producing scripts take the **internal** abbrev and key + `auto_generated//` off it directly. Passing `cdenJEC21` to them fails with + exit 2 and 271 bytes of stderr: + `Cannot open file .../auto_generated/cdenJEC21/datasetAndPresenterProps.conf: No such file or directory at .../JBrowseUtil.pm line 208`. + Nothing is missing — the generation step is fine, the two scripts families simply + disagree about which abbrev they consume. Any tool must resolve public→internal once + and feed each script the abbrev it expects. + +3. **Zero-track results verified as genuine, not broken.** `cneoJEC21` returns 0 ChIP-Seq + and 0 DNA-Seq tracks. Its `datasetAndPresenterProps.conf` contains the + `jbrowseChIPSeqBuildProps` and `jbrowseDnaSeqBuildProps` template anchors with **no + injected dataset entries beneath them**, whereas `tgonME49`'s carries real + `chipseq::…` rows. The organism has no such data loaded; the queries are working. + +4. **`jbrowseTracks` returning `tracks: []` is by design.** It emits the trackList + skeleton (`refSeqs`, `names`, `include`); the tracks arrive via the `include` list. Do + not treat its zero count as a failure. + +Not measured: `https://jbrestel.eupathdb.org/a/service/jbrowse/tracks/tgonME49/trackList.json` +307-redirects to EuPathDB autologin, so the live-site comparison was skipped. + ## 11. Deferred - **A real `apollo` application type.** When JBrowse2/Apollo3 arrives, `applicationType` From 8337a2328b46dbdbb32ca8acf5106b4b202c5670 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:52:02 -0400 Subject: [PATCH 25/42] fix(jbrowse): remove the dead refSeqs cache path that killed jbrowseRefSeqs e0e9a61bb ("comment out stuff to do with CACHE") commented out the getCacheFile/setCacheFile accessors and the setCacheFileName() call in new(), but left printFromCache() and setCacheFileName() calling them. Perl resolves methods at run time, so nothing complained until the script ran: jbrowseRefSeqs died at its first statement with Can't locate object method "getCacheFile" via package ApiCommonModel::Model::JBrowseUtil at .../JBrowseUtil.pm line 419 exit 255, 211 bytes on stderr, for every organism and both types since 2025-02-26. These scripts are served through responseFromCommand, which merges stderr into the JSON response body, so this was corrupting the live payload rather than only logging. Removed rather than restored. The disable was deliberate; the only caller of printFromCache is jbrowseRefSeqs, and the only caller of setCacheFileName was the line new() already commented out -- so the whole path is unreachable and restoring an accessor would merely re-arm dead code. Worse, the cache it would re-arm has no invalidation: it wrote _refSeqsCache.json into GUS_HOME once and returned it forever, outliving the build whose sequence names and lengths it held. The queries are two indexed lookups; there is nothing here worth caching incorrectly. Model/t/jbrowseutil.t pins it: every $jbrowseUtil->method() call in the shipped Model/bin/jbrowse* scripts must resolve against the package (scanned, not hand-listed, so a new script is covered for free), and the four cache subs must stay absent. Measured after installing, model UniDB, build 71, zero bytes on stderr and exit 0 in all six runs: tgonME49 genomic 2265 entries protein 8204 entries pfal3D7 genomic 16 entries protein 5389 entries cneoJEC21 genomic 14 entries protein 6862 entries Each entry is {name, start, end, length}. Suite: 188 tests, all pass. Co-Authored-By: Claude Opus 5 --- Model/bin/jbrowseRefSeqs | 10 +------ Model/lib/perl/JBrowseUtil.pm | 36 ------------------------ Model/t/jbrowseutil.t | 52 +++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 Model/t/jbrowseutil.t diff --git a/Model/bin/jbrowseRefSeqs b/Model/bin/jbrowseRefSeqs index cd4de5479c..91a65b9e28 100644 --- a/Model/bin/jbrowseRefSeqs +++ b/Model/bin/jbrowseRefSeqs @@ -11,13 +11,7 @@ use Data::Dumper; # TODO: remove gusHome from command line arg my ($gusHome, $projectName, $organismAbbrev, $type) = @ARGV; -my $jbrowseUtil = ApiCommonModel::Model::JBrowseUtil->new({projectName => $projectName, organismAbbrev => $organismAbbrev, fileName => "_refSeqsCache.json", type => $type }); - -# we're done if we can get from cache file -if($jbrowseUtil->printFromCache()) { - exit 0; -} -open(CACHE, "> " . $jbrowseUtil->getCacheFile()) or die "Cannot open file " . $jbrowseUtil->getCacheFile() . " for writing: $!"; +my $jbrowseUtil = ApiCommonModel::Model::JBrowseUtil->new({projectName => $projectName, organismAbbrev => $organismAbbrev, type => $type }); my $dbh = $jbrowseUtil->getDbh(); @@ -66,6 +60,4 @@ while(my ($sourceId, $length) = $sh->fetchrow_array()) { $dbh->disconnect(); print encode_json($result); -print CACHE encode_json($result); -close CACHE; diff --git a/Model/lib/perl/JBrowseUtil.pm b/Model/lib/perl/JBrowseUtil.pm index 8bff7c6b4b..43ad573143 100644 --- a/Model/lib/perl/JBrowseUtil.pm +++ b/Model/lib/perl/JBrowseUtil.pm @@ -31,9 +31,6 @@ sub getDbh { #my $datasetAndPresenterPropertiesBaseName = "datasetAndPresenterProps.conf"; # getters and setters for all class attributes -#sub getCacheFile {$_[0]->{_cache_file}} -#sub setCacheFile {$_[0]->{_cache_file} = $_[1]} - sub getFileName {$_[0]->{_fileName}} sub setFileName {$_[0]->{_fileName} = $_[1]} @@ -98,11 +95,6 @@ my @tester = %{$args}; my $type = lc($args->{type}) eq 'protein' ? 'protein' : 'genome'; $self->setType($type); - - # this one is a bit odd because it calls a function that calls the setter - # not sure if you need this - # $self->setCacheFileName(); - #my $organismAbbrev ="tgonME49"; # $args->{organismAbbrev};;$self->getOrganismAbbrev(); my $organismAbbrev = $self->getOrganismAbbrev(); @@ -226,20 +218,6 @@ sub makeProperties { -# this function makes the cacheFileName and sets it as a class attribute -sub setCacheFileName { - my ($self) = @_; - - my $organismAbbrev = $self->getOrganismAbbrev(); - my $fileName = $self->getFileName(); - - my $cacheFile = $self->getType() eq 'protein' - ? $ENV{GUS_HOME} . "/lib/jbrowse/auto_generated/$organismAbbrev/aa/$fileName" - : $ENV{GUS_HOME} . "/lib/jbrowse/auto_generated/$organismAbbrev/$fileName"; - - $self->setCacheFile($cacheFile); -} - # didn't touch anything below here sub toNestedHash { @@ -413,18 +391,4 @@ sub intronJunctionsQueryParams { return $querieParams->{default}->{$level}; } -sub printFromCache { - my ($self) = @_; - - my $cacheFile = $self->getCacheFile(); - - if(-e $cacheFile) { - open(FILE, $cacheFile) or die "Cannot open file $cacheFile for reading: $!"; - print ; - close FILE; - return 1 - } - return 0; -} - 1; diff --git a/Model/t/jbrowseutil.t b/Model/t/jbrowseutil.t new file mode 100644 index 0000000000..70606c365e --- /dev/null +++ b/Model/t/jbrowseutil.t @@ -0,0 +1,52 @@ +use strict; +use warnings; +use Test::More tests => 6; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::JBrowseUtil; + +my $U = 'ApiCommonModel::Model::JBrowseUtil'; + +# This file guards ONE failure mode, the one that killed jbrowseRefSeqs for a +# year and a half: an accessor is removed from JBrowseUtil while a caller keeps +# calling it. Perl resolves methods at run time, so nothing -- not compilation, +# not `perl -c`, not any other test here -- notices. The script dies at its +# first line of real work with `Can't locate object method`. +# +# It matters more than an ordinary missing-method bug because these scripts are +# served through responseFromCommand, which merges stderr into the JSON response +# body. A dead method is a corrupt payload on the live site, not a log line. + +# Every method invoked on a JBrowseUtil instance by the shipped jbrowse* +# scripts must exist on the package. Scanning the scripts rather than listing +# the methods here is deliberate: a new script gets covered for free, and the +# assertion cannot drift out of date the way a hand-kept list would. +my @scripts = sort glob('Model/bin/jbrowse*'); +ok(scalar(@scripts) >= 9, 'found the jbrowse* scripts to scan') or diag("cwd must be the ApiCommonModel checkout root"); + +my @missing; +foreach my $script (@scripts) { + open(my $fh, '<', $script) or die "Cannot read $script: $!"; + local $/; + my $src = <$fh>; + close $fh; + + next unless $src =~ /\Q$U\E->new/; + + my %seen; + while ($src =~ /\$jbrowseUtil\s*->\s*(\w+)\s*\(/g) { + next if $seen{$1}++; + push @missing, "$script: \$jbrowseUtil->$1()" unless $U->can($1); + } +} +is(join("\n", @missing), '', 'every JBrowseUtil method the jbrowse* scripts call exists') + or diag("removed accessor still has callers -- the script dies at run time with 'Can't locate object method'"); + +# The caching path was retired by e0e9a61bb ("comment out stuff to do with +# CACHE"), which disabled the setter call in new() but left the callers behind. +# Restoring the accessors would have re-armed a cache that has no invalidation: +# _refSeqsCache.json is written once into GUS_HOME and then returned forever, +# outliving the build whose sequence lengths it holds. It is gone, and these +# assertions exist so a future reader does not "restore" it by reflex. +foreach my $method (qw(getCacheFile setCacheFile setCacheFileName printFromCache)) { + ok(!$U->can($method), "$method is gone, not re-armed: the refSeqs cache had no invalidation"); +} From 648d9bd6865dd4d5a72b0a8e205a2691a67ccb08 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:54:54 -0400 Subject: [PATCH 26/42] feat(apollo): carry the internal organism abbrev alongside the public one apidb.organism holds two identifiers and the jbrowse scripts disagree about which they consume: jbrowseTracks selects `where public_abbrev = ?` and returns the internal one, while the five track producers (jbrowseRnaAndChipSeqTracks, jbrowseRNASeqJunctionTracks, jbrowseOrganismSpecificTracks, jbrowseDNASeqTracks) take the internal abbrev and key auto_generated// off it. Feeding a producer the public abbrev gives exit 2 on a missing datasetAndPresenterProps.conf. jbrowseOrganismList now also selects `o.abbrev as internal_abbrev` and Portal carries it through normalise, so the release tool can resolve public -> internal once and hand each script the abbrev it expects. organism_abbrev stays the PUBLIC abbrev and remains the hash key: the Apollo roster, the overlay file and reconciliation all key on it, and it is what the portal shows. Nothing existing moved. Measured on live UniDB, build 71, through Portal->loadFromCommand: 831 organisms, exit 0, zero bytes on stderr, zero warnings, and internal_abbrev defined for all 831. **37 organisms have internal_abbrev ne abbrev** -- that is the live population where the trap bites, not just the one Cryptococcus. Several are cross-genus renames (nglaCBS138 -> cglaCBS138, pory70-15 -> mory70-15) and one is mere case (scerS288C -> scerS288c), which no eyeball would catch. Model/t/fixtures/portal.json regenerated from the live command rather than hand-edited, same nine organisms. Two of them now exercise the split: cdenJEC21 (cneoJEC21) and nglaCBS138 (cglaCBS138). portal.t pins cdenJEC21's internal_abbrev and that its public abbrev is unchanged. Suite: 191 tests, all pass. Co-Authored-By: Claude Opus 5 --- Model/bin/jbrowseOrganismList | 1 + Model/lib/perl/ApolloRelease/Portal.pm | 13 +- Model/t/fixtures/portal.json | 961 +++++++++++++------------ Model/t/portal.t | 21 +- 4 files changed, 517 insertions(+), 479 deletions(-) diff --git a/Model/bin/jbrowseOrganismList b/Model/bin/jbrowseOrganismList index 7ed66c26c4..35ab103d8c 100644 --- a/Model/bin/jbrowseOrganismList +++ b/Model/bin/jbrowseOrganismList @@ -12,6 +12,7 @@ my $jbrowseUtil = ApiCommonModel::Model::JBrowseUtil->new({projectName => $proje my $dbh = $jbrowseUtil->getDbh(); my $sql = "select distinct o.public_abbrev as organism_abbrev + , o.abbrev as internal_abbrev , o.name_for_filenames , o.strain_abbrev , o.IS_REFERENCE_STRAIN diff --git a/Model/lib/perl/ApolloRelease/Portal.pm b/Model/lib/perl/ApolloRelease/Portal.pm index 0623354cef..5d19f45777 100644 --- a/Model/lib/perl/ApolloRelease/Portal.pm +++ b/Model/lib/perl/ApolloRelease/Portal.pm @@ -9,8 +9,16 @@ use Scalar::Util qw(looks_like_number); # Reads the organism list produced by Model/bin/jbrowseOrganismList and # normalises it. Keys arrive lowercase from DBD::Pg; booleans arrive as the -# strings "1"/"0". Everything downstream sees plain 0/1 and a single -# `abbrev` key. +# strings "1"/"0". Everything downstream sees plain 0/1 and an `abbrev` key. +# +# TWO abbrevs come through, and they are not interchangeable. `abbrev` is +# apidb.organism.public_abbrev -- the hash key, what the portal shows, and what +# jbrowseTracks matches on. `internal_abbrev` is apidb.organism.abbrev, which +# the five jbrowse track producers use to find auto_generated//. They +# are the same string for all but a handful of renamed organisms (cdenJEC21 / +# cneoJEC21), which is exactly why passing the wrong one is easy and fails far +# from the mistake -- exit 2 on a missing datasetAndPresenterProps.conf, or a +# query that quietly matches nothing. # Warnings are COLLECTED, never written to stderr. loadFromCommand treats any # stderr byte from its child as proof that child's output is untrustworthy, and @@ -92,6 +100,7 @@ sub normalise { $byAbbrev{$abbrev} = { abbrev => $abbrev, + internal_abbrev => $raw->{internal_abbrev}, name => $raw->{name}, name_for_filenames => $raw->{name_for_filenames}, strain_abbrev => $raw->{strain_abbrev}, diff --git a/Model/t/fixtures/portal.json b/Model/t/fixtures/portal.json index 228a98d799..b2c7cdd6d0 100644 --- a/Model/t/fixtures/portal.json +++ b/Model/t/fixtures/portal.json @@ -1,871 +1,880 @@ { "organisms": [ { + "species_ncbi_tax_id": "5811", + "is_reference_strain": "1", + "name": "Toxoplasma gondii ME49", + "internal_abbrev": "tgonME49", + "strain_abbrev": "ME49", + "is_annotated_genome": "1", + "name_for_filenames": "TgondiiME49", + "organism_abbrev": "tgonME49", "history": [ { - "public_abbrev": "cdenJEC21", - "build_number": "51", + "public_abbrev": "tgonME49", + "annotation_version": "Jul 01, 2023", "genome_source": "GenBank", - "annotation_version": "Jun 16, 2016", - "genome_version": "GCA_000091045.1", - "annotation_source": "GenBank" + "genome_version": "GCF_000006565.2", + "annotation_source": "Apollo", + "build_number": "71" }, { + "build_number": "70", "genome_source": "GenBank", - "annotation_version": "Jun 16, 2016", - "build_number": "48", - "genome_version": "Jun 16, 2016", - "public_abbrev": "cdenJEC21", - "annotation_source": "GenBank" + "annotation_source": "Apollo", + "genome_version": "GCF_000006565.2", + "annotation_version": "Jul 01, 2023", + "public_abbrev": "tgonME49" }, { - "annotation_source": "GenBank", - "public_abbrev": "cdenJEC21", + "annotation_version": "Jul 01, 2023", + "public_abbrev": "tgonME49", + "build_number": "69", "genome_source": "GenBank", - "annotation_version": "Jun 16, 2016", - "build_number": "44", - "genome_version": "Jun 16, 2016" + "annotation_source": "Apollo", + "genome_version": "GCF_000006565.2" }, { - "annotation_source": "GenBank", - "public_abbrev": "cdenJEC21", - "build_number": "43", - "annotation_version": "Jun 16, 2016", + "annotation_version": "Jul 01, 2023", + "public_abbrev": "tgonME49", + "build_number": "68", + "annotation_source": "Apollo", + "genome_version": "GCF_000006565.2", + "genome_source": "GenBank" + }, + { + "public_abbrev": "tgonME49", + "annotation_version": "Jul 01, 2023", "genome_source": "GenBank", - "genome_version": "Jun 16, 2016" + "annotation_source": "Apollo", + "genome_version": "GCF_000006565.2", + "build_number": "67" }, { - "genome_version": "Jun 16, 2016", + "annotation_version": "Jul 01, 2023", + "public_abbrev": "tgonME49", + "build_number": "66", + "annotation_source": "Apollo", + "genome_version": "GCF_000006565.2", + "genome_source": "GenBank" + }, + { + "annotation_version": "Mar 22, 2015", + "public_abbrev": "tgonME49", + "build_number": "65", + "annotation_source": "GenBank", + "genome_version": "GCF_000006565.2", + "genome_source": "GenBank" + }, + { + "build_number": "64", + "annotation_source": "GenBank", + "genome_version": "GCF_000006565.2", "genome_source": "GenBank", - "annotation_version": "Jun 16, 2016", - "build_number": "41", - "public_abbrev": "cdenJEC21", - "annotation_source": "GenBank" + "annotation_version": "Mar 22, 2015", + "public_abbrev": "tgonME49" }, { + "annotation_version": "Mar 22, 2015", + "public_abbrev": "tgonME49", + "build_number": "49", + "genome_version": "GCF_000006565.2", "annotation_source": "GenBank", - "public_abbrev": "cdenJEC21", + "genome_source": "GenBank" + }, + { + "annotation_version": "Mar 22, 2015", + "public_abbrev": "tgonME49", + "build_number": "29", + "genome_version": "Nov 01, 2013", + "annotation_source": "GenBank", + "genome_source": "GenBank" + }, + { + "genome_source": "JCVI", + "genome_version": "Apr 23, 2013", + "annotation_source": "JCVI", + "build_number": "19", + "public_abbrev": "tgonME49", + "annotation_version": "Apr 23, 2013" + }, + { + "public_abbrev": "tgonME49", + "annotation_version": "Jul 8, 2012", + "annotation_source": "JCVI", + "genome_version": "Jul 8, 2012", + "genome_source": "JCVI", + "build_number": "16" + }, + { + "annotation_version": "Jul 23, 2008", + "public_abbrev": "tgonME49", + "build_number": "0", + "genome_source": "JCVI", + "genome_version": "Jul 23, 2008", + "annotation_source": "JCVI" + } + ] + }, + { + "name_for_filenames": "TreeseiQM6a", + "history": [ + { + "annotation_version": "Nov 01, 2021", + "public_abbrev": "treeQM6a", + "build_number": "56", "genome_source": "GenBank", - "annotation_version": "Jun 16, 2016", - "build_number": "37", - "genome_version": "Jun 16, 2016" + "genome_version": "GCA_000167675.2", + "annotation_source": "Apollo" }, { + "build_number": "51", + "genome_version": "GCA_000167675.2", "annotation_source": "GenBank", - "public_abbrev": "cdenJEC21", - "annotation_version": "Jun 16, 2016", "genome_source": "GenBank", - "build_number": "36", - "genome_version": "Jun 16, 2016" + "annotation_version": "Dec 13, 2012", + "public_abbrev": "treeQM6a" }, { - "genome_version": "Jun 16, 2016", "build_number": "29", - "annotation_version": "Jun 16, 2016", + "genome_version": "Mar 14, 2015", + "annotation_source": "GenBank", "genome_source": "GenBank", - "public_abbrev": "cdenJEC21", - "annotation_source": "GenBank" + "annotation_version": "Dec 13, 2012", + "public_abbrev": "treeQM6a" }, { - "genome_source": "GenBank", - "annotation_version": "2010-08-14", - "build_number": "15", - "genome_version": "2010-08-14", - "public_abbrev": "cdenJEC21", - "annotation_source": "TIGR-Stanford" + "genome_source": "JGI", + "annotation_source": "JGI", + "genome_version": "2", + "build_number": "19", + "public_abbrev": "treeQM6a", + "annotation_version": "2" } ], + "organism_abbrev": "treeQM6a", + "species_ncbi_tax_id": "51453", "is_reference_strain": "1", - "species_ncbi_tax_id": "40410", - "strain_abbrev": "JEC21", - "name_for_filenames": "CdeneoformansJEC21", + "internal_abbrev": "treeQM6a", + "name": "Trichoderma reesei QM6a", + "strain_abbrev": "QM6a", + "is_annotated_genome": "1" + }, + { + "strain_abbrev": "Lister427_2018", "is_annotated_genome": "1", - "name": "Cryptococcus deneoformans JEC21", - "organism_abbrev": "cdenJEC21" + "species_ncbi_tax_id": "5691", + "is_reference_strain": "0", + "name": "Trypanosoma brucei Lister strain 427 2018", + "internal_abbrev": "tbruLister427_2018", + "organism_abbrev": "tbruLister427_2018", + "history": [ + { + "build_number": "52", + "genome_source": "GenBank", + "genome_version": "GCA_900497135.1", + "annotation_source": "SiegelLab", + "annotation_version": "Dec 17, 2018", + "public_abbrev": "tbruLister427_2018" + }, + { + "build_number": "42", + "genome_source": "SiegelLab", + "annotation_source": "SiegelLab", + "genome_version": "Dec 17, 2018", + "annotation_version": "Dec 17, 2018", + "public_abbrev": "tbruLister427_2018" + } + ], + "name_for_filenames": "TbruceiLister427_2018" }, { - "is_reference_strain": "1", "history": [ { - "public_abbrev": "pfal3D7", "build_number": "71", - "annotation_version": "Sep 01, 2020", - "genome_source": "GenBank", + "annotation_source": "GeneDB", "genome_version": "GCA_000002765.3", - "annotation_source": "GeneDB" + "genome_source": "GenBank", + "annotation_version": "Sep 01, 2020", + "public_abbrev": "pfal3D7" }, { "public_abbrev": "pfal3D7", "annotation_version": "Sep 01, 2020", "genome_source": "GenBank", - "build_number": "69", "genome_version": "GCA_000002765.3", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "build_number": "69" }, { - "public_abbrev": "pfal3D7", + "build_number": "68", + "genome_source": "GenBank", + "annotation_source": "GeneDB", "genome_version": "GCA_000002765.3", "annotation_version": "Sep 01, 2020", - "genome_source": "GenBank", - "build_number": "68", - "annotation_source": "GeneDB" + "public_abbrev": "pfal3D7" }, { + "annotation_version": "Sep 01, 2020", "public_abbrev": "pfal3D7", "build_number": "66", - "genome_source": "GenBank", - "annotation_version": "Sep 01, 2020", + "annotation_source": "GeneDB", "genome_version": "GCA_000002765.3", - "annotation_source": "GeneDB" + "genome_source": "GenBank" }, { + "public_abbrev": "pfal3D7", "annotation_version": "Sep 01, 2020", - "genome_source": "GenBank", - "build_number": "63", "genome_version": "GCA_000002765.3", - "public_abbrev": "pfal3D7", - "annotation_source": "GeneDB" - }, - { "annotation_source": "GeneDB", "genome_source": "GenBank", + "build_number": "63" + }, + { "annotation_version": "Sep 01, 2020", + "public_abbrev": "pfal3D7", "build_number": "62", - "genome_version": "GCA_000002765.3", - "public_abbrev": "pfal3D7" + "genome_source": "GenBank", + "annotation_source": "GeneDB", + "genome_version": "GCA_000002765.3" }, { - "public_abbrev": "pfal3D7", - "genome_version": "GCA_000002765.3", - "annotation_version": "Sep 01, 2020", "genome_source": "GenBank", + "annotation_source": "GeneDB", + "genome_version": "GCA_000002765.3", "build_number": "60", - "annotation_source": "GeneDB" + "public_abbrev": "pfal3D7", + "annotation_version": "Sep 01, 2020" }, { + "genome_source": "GenBank", "genome_version": "GCA_000002765.3", + "annotation_source": "GeneDB", "build_number": "57", - "annotation_version": "Sep 01, 2020", - "genome_source": "GenBank", "public_abbrev": "pfal3D7", - "annotation_source": "GeneDB" + "annotation_version": "Sep 01, 2020" }, { - "annotation_source": "GeneDB", - "build_number": "55", "annotation_version": "Sep 01, 2020", - "genome_source": "GenBank", + "public_abbrev": "pfal3D7", + "build_number": "55", "genome_version": "GCA_000002765.3", - "public_abbrev": "pfal3D7" + "annotation_source": "GeneDB", + "genome_source": "GenBank" }, { "build_number": "53", + "genome_version": "GCA_000002765.3", + "annotation_source": "GeneDB", "genome_source": "GenBank", "annotation_version": "Sep 01, 2020", - "genome_version": "GCA_000002765.3", - "public_abbrev": "pfal3D7", - "annotation_source": "GeneDB" + "public_abbrev": "pfal3D7" }, { + "build_number": "51", + "annotation_source": "GeneDB", "genome_version": "GCA_000002765.3", "genome_source": "GenBank", "annotation_version": "Sep 01, 2020", - "build_number": "51", + "public_abbrev": "pfal3D7" + }, + { + "annotation_version": "Jun 18, 2015", "public_abbrev": "pfal3D7", + "build_number": "48", + "genome_source": "GeneDB", + "genome_version": "Jun 18, 2015", "annotation_source": "GeneDB" }, { + "genome_source": "GeneDB", "annotation_source": "GeneDB", - "public_abbrev": "pfal3D7", "genome_version": "Jun 18, 2015", - "annotation_version": "Jun 18, 2015", - "genome_source": "GeneDB", - "build_number": "48" + "build_number": "43", + "public_abbrev": "pfal3D7", + "annotation_version": "Jun 18, 2015" }, { + "genome_version": "Jun 18, 2015", "annotation_source": "GeneDB", - "build_number": "43", - "annotation_version": "Jun 18, 2015", "genome_source": "GeneDB", - "genome_version": "Jun 18, 2015", - "public_abbrev": "pfal3D7" + "build_number": "41", + "public_abbrev": "pfal3D7", + "annotation_version": "Jun 18, 2015" }, { - "annotation_source": "GeneDB", + "annotation_version": "Jun 18, 2015", "public_abbrev": "pfal3D7", + "build_number": "37", + "annotation_source": "GeneDB", "genome_version": "Jun 18, 2015", - "annotation_version": "Jun 18, 2015", - "genome_source": "GeneDB", - "build_number": "41" + "genome_source": "GeneDB" }, { + "genome_source": "GeneDB", + "genome_version": "Jun 18, 2015", "annotation_source": "GeneDB", + "build_number": "35", "public_abbrev": "pfal3D7", - "genome_version": "Jun 18, 2015", - "build_number": "37", - "genome_source": "GeneDB", "annotation_version": "Jun 18, 2015" }, { - "annotation_source": "GeneDB", - "build_number": "35", - "genome_source": "GeneDB", + "public_abbrev": "pfal3D7", "annotation_version": "Jun 18, 2015", "genome_version": "Jun 18, 2015", - "public_abbrev": "pfal3D7" - }, - { "annotation_source": "GeneDB", "genome_source": "GeneDB", - "annotation_version": "Jun 18, 2015", - "build_number": "32", - "genome_version": "Jun 18, 2015", - "public_abbrev": "pfal3D7" + "build_number": "32" }, { + "annotation_version": "Jun 18, 2015", + "public_abbrev": "pfal3D7", + "build_number": "29", "annotation_source": "GeneDB", "genome_version": "Jun 18, 2015", - "build_number": "29", - "genome_source": "GeneDB", - "annotation_version": "Jun 18, 2015", - "public_abbrev": "pfal3D7" + "genome_source": "GeneDB" }, { - "annotation_source": "GeneDB", - "genome_source": "GeneDB", + "public_abbrev": "pfal3D7", "annotation_version": "Mar 1, 2013", - "build_number": "26", + "genome_source": "GeneDB", + "annotation_source": "GeneDB", "genome_version": "Mar 1, 2013", - "public_abbrev": "pfal3D7" + "build_number": "26" }, { + "annotation_version": "Mar 1, 2013", "public_abbrev": "pfal3D7", "build_number": "19", "genome_source": "GeneDB", - "annotation_version": "Mar 1, 2013", - "genome_version": "Mar 1, 2013", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "genome_version": "Mar 1, 2013" }, { + "public_abbrev": "pfal3D7", + "annotation_version": "Feb 1, 2012", + "genome_version": "Feb 1, 2012", "annotation_source": "GeneDB", - "public_abbrev": "pfal3D7", - "build_number": "14", "genome_source": "GeneDB", - "annotation_version": "Feb 1, 2012", - "genome_version": "Feb 1, 2012" + "build_number": "14" }, { - "public_abbrev": "pfal3D7", + "build_number": "0", + "genome_source": "n/a", "genome_version": "n/a", + "annotation_source": "n/a", "annotation_version": "n/a", - "genome_source": "n/a", - "build_number": "0", - "annotation_source": "n/a" + "public_abbrev": "pfal3D7" } ], - "species_ncbi_tax_id": "5833", - "is_annotated_genome": "1", + "organism_abbrev": "pfal3D7", "name_for_filenames": "Pfalciparum3D7", + "is_annotated_genome": "1", "strain_abbrev": "3D7", - "organism_abbrev": "pfal3D7", - "name": "Plasmodium falciparum 3D7" + "name": "Plasmodium falciparum 3D7", + "internal_abbrev": "pfal3D7", + "is_reference_strain": "1", + "species_ncbi_tax_id": "5833" }, { - "name_for_filenames": "NglabratusCBS138", + "species_ncbi_tax_id": "5478", + "is_reference_strain": "1", + "internal_abbrev": "cglaCBS138", + "name": "Nakaseomyces glabratus CBS 138", "strain_abbrev": "CBS138", "is_annotated_genome": "1", - "name": "Nakaseomyces glabratus CBS 138", + "name_for_filenames": "NglabratusCBS138", "organism_abbrev": "nglaCBS138", "history": [ { + "genome_source": "GenBank", "annotation_source": "CGD", "genome_version": "GCA_000002545.2", "build_number": "69", - "annotation_version": "s02-m07-r27", - "genome_source": "GenBank", - "public_abbrev": "nglaCBS138" + "public_abbrev": "nglaCBS138", + "annotation_version": "s02-m07-r27" }, { - "annotation_source": "CGD", - "public_abbrev": "nglaCBS138", "genome_source": "GenBank", - "annotation_version": "s02-m07-r27", + "genome_version": "GCA_000002545.2", + "annotation_source": "CGD", "build_number": "64", - "genome_version": "GCA_000002545.2" + "public_abbrev": "nglaCBS138", + "annotation_version": "s02-m07-r27" }, { - "annotation_source": "CGD", + "annotation_version": "s02-m07-r27", "public_abbrev": "nglaCBS138", "build_number": "53", - "annotation_version": "s02-m07-r27", "genome_source": "GenBank", + "annotation_source": "CGD", "genome_version": "GCA_000002545.2" }, { - "annotation_source": "CGD", + "public_abbrev": "nglaCBS138", "annotation_version": "s02-m07-r27", "genome_source": "GenBank", - "build_number": "51", + "annotation_source": "CGD", "genome_version": "GCA_000002545.2", - "public_abbrev": "nglaCBS138" + "build_number": "51" }, { - "annotation_source": "CGD", - "public_abbrev": "nglaCBS138", "build_number": "39", "genome_source": "CGD", + "annotation_source": "CGD", + "genome_version": "s02-m07-r27", "annotation_version": "s02-m07-r27", - "genome_version": "s02-m07-r27" + "public_abbrev": "nglaCBS138" }, { - "annotation_version": "s02-m07-r06", "genome_source": "CGD", - "build_number": "29", "genome_version": "s02-m07-r06", + "annotation_source": "CGD", + "build_number": "29", "public_abbrev": "nglaCBS138", - "annotation_source": "CGD" + "annotation_version": "s02-m07-r06" }, { - "annotation_source": "CGD", "public_abbrev": "nglaCBS138", "annotation_version": "Nov 17, 2013", "genome_source": "CGD", - "build_number": "21", - "genome_version": "Nov 17, 2013" - } - ], - "is_reference_strain": "1", - "species_ncbi_tax_id": "5478" - }, - { - "history": [ - { - "annotation_source": "Ensembl", - "genome_source": "Ensembl", - "annotation_version": "GRCh38.p13", - "build_number": "52", - "genome_version": "GCA_000001405.28", - "public_abbrev": "hsapREF" - }, - { - "build_number": "29", - "annotation_version": "24", - "genome_source": "GENCODE", - "genome_version": "GRCh38", - "public_abbrev": "hsapREF", - "annotation_source": "GENCODE" - }, - { - "annotation_source": "GENCODE", - "public_abbrev": "hsapREF", - "annotation_version": "13", - "genome_source": "UCSC", - "build_number": "18", - "genome_version": "hg19" + "genome_version": "Nov 17, 2013", + "annotation_source": "CGD", + "build_number": "21" } - ], - "is_reference_strain": "1", - "species_ncbi_tax_id": "9606", - "strain_abbrev": "REF", - "name_for_filenames": "HsapiensREF", - "is_annotated_genome": "1", - "name": "Homo sapiens REF", - "organism_abbrev": "hsapREF" + ] }, { - "is_annotated_genome": "1", - "strain_abbrev": "ME49", - "name_for_filenames": "TgondiiME49", - "organism_abbrev": "tgonME49", - "name": "Toxoplasma gondii ME49", - "is_reference_strain": "1", + "name_for_filenames": "CdeneoformansJEC21", + "organism_abbrev": "cdenJEC21", "history": [ { - "annotation_version": "Jul 01, 2023", - "genome_source": "GenBank", - "build_number": "71", - "genome_version": "GCF_000006565.2", - "public_abbrev": "tgonME49", - "annotation_source": "Apollo" - }, - { - "annotation_source": "Apollo", - "annotation_version": "Jul 01, 2023", - "genome_source": "GenBank", - "build_number": "70", - "genome_version": "GCF_000006565.2", - "public_abbrev": "tgonME49" + "annotation_version": "Jun 16, 2016", + "public_abbrev": "cdenJEC21", + "build_number": "51", + "genome_version": "GCA_000091045.1", + "annotation_source": "GenBank", + "genome_source": "GenBank" }, { - "annotation_source": "Apollo", - "public_abbrev": "tgonME49", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016", + "annotation_source": "GenBank", + "genome_version": "Jun 16, 2016", "genome_source": "GenBank", - "annotation_version": "Jul 01, 2023", - "build_number": "69", - "genome_version": "GCF_000006565.2" + "build_number": "48" }, { - "genome_version": "GCF_000006565.2", - "build_number": "68", - "annotation_version": "Jul 01, 2023", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016", + "annotation_source": "GenBank", + "genome_version": "Jun 16, 2016", "genome_source": "GenBank", - "public_abbrev": "tgonME49", - "annotation_source": "Apollo" + "build_number": "44" }, { + "genome_version": "Jun 16, 2016", + "annotation_source": "GenBank", "genome_source": "GenBank", - "annotation_version": "Jul 01, 2023", - "build_number": "67", - "genome_version": "GCF_000006565.2", - "public_abbrev": "tgonME49", - "annotation_source": "Apollo" + "build_number": "43", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016" }, { - "public_abbrev": "tgonME49", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016", + "annotation_source": "GenBank", + "genome_version": "Jun 16, 2016", "genome_source": "GenBank", - "annotation_version": "Jul 01, 2023", - "build_number": "66", - "genome_version": "GCF_000006565.2", - "annotation_source": "Apollo" + "build_number": "41" }, { - "annotation_version": "Mar 22, 2015", + "annotation_version": "Jun 16, 2016", + "public_abbrev": "cdenJEC21", + "build_number": "37", "genome_source": "GenBank", - "build_number": "65", - "genome_version": "GCF_000006565.2", - "public_abbrev": "tgonME49", - "annotation_source": "GenBank" + "annotation_source": "GenBank", + "genome_version": "Jun 16, 2016" }, { - "genome_version": "GCF_000006565.2", - "build_number": "64", + "public_abbrev": "cdenJEC21", + "annotation_version": "Jun 16, 2016", "genome_source": "GenBank", - "annotation_version": "Mar 22, 2015", - "public_abbrev": "tgonME49", - "annotation_source": "GenBank" - }, - { + "genome_version": "Jun 16, 2016", "annotation_source": "GenBank", - "genome_version": "GCF_000006565.2", - "genome_source": "GenBank", - "annotation_version": "Mar 22, 2015", - "build_number": "49", - "public_abbrev": "tgonME49" + "build_number": "36" }, { - "genome_version": "Nov 01, 2013", - "annotation_version": "Mar 22, 2015", - "genome_source": "GenBank", + "annotation_version": "Jun 16, 2016", + "public_abbrev": "cdenJEC21", "build_number": "29", - "public_abbrev": "tgonME49", - "annotation_source": "GenBank" - }, - { - "annotation_version": "Apr 23, 2013", - "genome_source": "JCVI", - "build_number": "19", - "genome_version": "Apr 23, 2013", - "public_abbrev": "tgonME49", - "annotation_source": "JCVI" - }, - { - "public_abbrev": "tgonME49", - "genome_version": "Jul 8, 2012", - "genome_source": "JCVI", - "annotation_version": "Jul 8, 2012", - "build_number": "16", - "annotation_source": "JCVI" + "annotation_source": "GenBank", + "genome_version": "Jun 16, 2016", + "genome_source": "GenBank" }, { - "genome_version": "Jul 23, 2008", - "genome_source": "JCVI", - "annotation_version": "Jul 23, 2008", - "build_number": "0", - "public_abbrev": "tgonME49", - "annotation_source": "JCVI" + "annotation_version": "2010-08-14", + "public_abbrev": "cdenJEC21", + "build_number": "15", + "genome_source": "GenBank", + "genome_version": "2010-08-14", + "annotation_source": "TIGR-Stanford" } ], - "species_ncbi_tax_id": "5811" + "name": "Cryptococcus deneoformans JEC21", + "internal_abbrev": "cneoJEC21", + "species_ncbi_tax_id": "40410", + "is_reference_strain": "1", + "is_annotated_genome": "1", + "strain_abbrev": "JEC21" }, { "is_reference_strain": "1", + "species_ncbi_tax_id": "2059318", + "internal_abbrev": "hcapNAm1", + "name": "Histoplasma mississippiense (nom. inval.) NAm1", + "strain_abbrev": "NAm1", + "is_annotated_genome": "1", + "name_for_filenames": "HmississippienseNAm1", "history": [ { - "annotation_source": "GenBank", - "build_number": "51", - "genome_source": "GenBank", + "public_abbrev": "hcapNAm1", "annotation_version": "Mar 23, 2015", "genome_version": "GCA_000149585.1", - "public_abbrev": "hcapNAm1" + "annotation_source": "GenBank", + "genome_source": "GenBank", + "build_number": "51" }, { - "annotation_source": "GenBank", - "build_number": "29", + "public_abbrev": "hcapNAm1", "annotation_version": "Mar 23, 2015", - "genome_source": "GenBank", + "annotation_source": "GenBank", "genome_version": "Mar 23, 2015", - "public_abbrev": "hcapNAm1" + "genome_source": "GenBank", + "build_number": "29" }, { - "annotation_version": "1", - "genome_source": "GenBank", "build_number": "16", + "annotation_source": "Broad", "genome_version": "2009-08-10", - "public_abbrev": "hcapNAm1", - "annotation_source": "Broad" - } - ], - "species_ncbi_tax_id": "2059318", - "is_annotated_genome": "1", - "strain_abbrev": "NAm1", - "name_for_filenames": "HmississippienseNAm1", - "organism_abbrev": "hcapNAm1", - "name": "Histoplasma mississippiense (nom. inval.) NAm1" - }, - { - "is_reference_strain": "0", - "history": [ - { - "public_abbrev": "tbruLister427_2018", - "annotation_version": "Dec 17, 2018", "genome_source": "GenBank", - "build_number": "52", - "genome_version": "GCA_900497135.1", - "annotation_source": "SiegelLab" - }, - { - "annotation_source": "SiegelLab", - "genome_version": "Dec 17, 2018", - "build_number": "42", - "annotation_version": "Dec 17, 2018", - "genome_source": "SiegelLab", - "public_abbrev": "tbruLister427_2018" + "annotation_version": "1", + "public_abbrev": "hcapNAm1" } ], - "species_ncbi_tax_id": "5691", - "is_annotated_genome": "1", - "name_for_filenames": "TbruceiLister427_2018", - "strain_abbrev": "Lister427_2018", - "organism_abbrev": "tbruLister427_2018", - "name": "Trypanosoma brucei Lister strain 427 2018" + "organism_abbrev": "hcapNAm1" }, { - "organism_abbrev": "tbruTREU927", + "is_reference_strain": "1", + "species_ncbi_tax_id": "5691", + "internal_abbrev": "tbruTREU927", "name": "Trypanosoma brucei brucei TREU927", + "strain_abbrev": "TREU927", "is_annotated_genome": "1", "name_for_filenames": "TbruceiTREU927", - "strain_abbrev": "TREU927", - "species_ncbi_tax_id": "5691", - "is_reference_strain": "1", + "organism_abbrev": "tbruTREU927", "history": [ { - "annotation_source": "VEuPathDB", - "genome_version": "GCA_000002445.1", "genome_source": "GenBank", - "annotation_version": "Feb 23, 2024", + "genome_version": "GCA_000002445.1", + "annotation_source": "VEuPathDB", "build_number": "70", - "public_abbrev": "tbruTREU927" + "public_abbrev": "tbruTREU927", + "annotation_version": "Feb 23, 2024" }, { + "build_number": "69", "genome_version": "GCA_000002445.1", - "annotation_version": "Feb 23, 2024", + "annotation_source": "VEuPathDB", "genome_source": "GenBank", - "build_number": "69", - "public_abbrev": "tbruTREU927", - "annotation_source": "VEuPathDB" + "annotation_version": "Feb 23, 2024", + "public_abbrev": "tbruTREU927" }, { "annotation_source": "GeneDB", "genome_version": "GCA_000002445.1", - "annotation_version": "Nov 01, 2020", "genome_source": "GenBank", "build_number": "66", - "public_abbrev": "tbruTREU927" + "public_abbrev": "tbruTREU927", + "annotation_version": "Nov 01, 2020" }, { + "build_number": "65", "annotation_source": "GeneDB", "genome_version": "GCA_000002445.1", - "annotation_version": "Nov 01, 2020", "genome_source": "GenBank", - "build_number": "65", + "annotation_version": "Nov 01, 2020", "public_abbrev": "tbruTREU927" }, { - "genome_version": "GCA_000002445.1", - "build_number": "63", - "genome_source": "GenBank", - "annotation_version": "Nov 01, 2020", "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "annotation_version": "Nov 01, 2020", + "genome_source": "GenBank", + "genome_version": "GCA_000002445.1", + "annotation_source": "GeneDB", + "build_number": "63" }, { - "public_abbrev": "tbruTREU927", - "build_number": "60", "genome_source": "GenBank", - "annotation_version": "Nov 01, 2020", "genome_version": "GCA_000002445.1", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "build_number": "60", + "public_abbrev": "tbruTREU927", + "annotation_version": "Nov 01, 2020" }, { - "genome_version": "GCA_000002445.1", "annotation_version": "Nov 01, 2020", - "genome_source": "GenBank", - "build_number": "58", "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "build_number": "58", + "genome_source": "GenBank", + "annotation_source": "GeneDB", + "genome_version": "GCA_000002445.1" }, { "build_number": "57", "genome_source": "GenBank", - "annotation_version": "Nov 01, 2020", "genome_version": "GCA_000002445.1", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "annotation_version": "Nov 01, 2020", + "public_abbrev": "tbruTREU927" }, { - "genome_version": "GCA_000002445.1", - "build_number": "56", "annotation_version": "Nov 01, 2020", - "genome_source": "GenBank", "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "build_number": "56", + "genome_version": "GCA_000002445.1", + "annotation_source": "GeneDB", + "genome_source": "GenBank" }, { "public_abbrev": "tbruTREU927", + "annotation_version": "Nov 01, 2020", + "annotation_source": "GeneDB", "genome_version": "GCA_000002445.1", "genome_source": "GenBank", - "annotation_version": "Nov 01, 2020", - "build_number": "54", - "annotation_source": "GeneDB" + "build_number": "54" }, { - "annotation_source": "GeneDB", "public_abbrev": "tbruTREU927", "annotation_version": "Nov 01, 2020", + "genome_version": "GCA_000002445.1", + "annotation_source": "GeneDB", "genome_source": "GenBank", - "build_number": "52", - "genome_version": "GCA_000002445.1" + "build_number": "52" }, { + "public_abbrev": "tbruTREU927", "annotation_version": "Jul 10, 2018", - "genome_source": "GeneDB", - "build_number": "51", "genome_version": "Jul 10, 2018", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "genome_source": "GeneDB", + "build_number": "51" }, { + "build_number": "49", "annotation_source": "GeneDB", "genome_version": "Jul 10, 2018", - "build_number": "49", - "annotation_version": "Jul 10, 2018", "genome_source": "GeneDB", + "annotation_version": "Jul 10, 2018", "public_abbrev": "tbruTREU927" }, { - "public_abbrev": "tbruTREU927", + "annotation_source": "GeneDB", + "genome_version": "Jul 10, 2018", "genome_source": "GeneDB", - "annotation_version": "Jul 10, 2018", "build_number": "48", - "genome_version": "Jul 10, 2018", - "annotation_source": "GeneDB" + "public_abbrev": "tbruTREU927", + "annotation_version": "Jul 10, 2018" }, { - "annotation_source": "GeneDB", - "public_abbrev": "tbruTREU927", "genome_version": "Jul 10, 2018", + "annotation_source": "GeneDB", "genome_source": "GeneDB", - "annotation_version": "Jul 10, 2018", - "build_number": "46" + "build_number": "46", + "public_abbrev": "tbruTREU927", + "annotation_version": "Jul 10, 2018" }, { - "build_number": "45", + "public_abbrev": "tbruTREU927", "annotation_version": "Jul 10, 2018", "genome_source": "GeneDB", + "annotation_source": "GeneDB", "genome_version": "Jul 10, 2018", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "build_number": "45" }, { - "genome_source": "GeneDB", + "public_abbrev": "tbruTREU927", "annotation_version": "Jul 10, 2018", - "build_number": "44", + "annotation_source": "GeneDB", "genome_version": "Jul 10, 2018", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "genome_source": "GeneDB", + "build_number": "44" }, { - "annotation_source": "GeneDB", - "public_abbrev": "tbruTREU927", "genome_source": "GeneDB", - "annotation_version": "Jul 10, 2018", + "genome_version": "Jul 10, 2018", + "annotation_source": "GeneDB", "build_number": "43", - "genome_version": "Jul 10, 2018" + "public_abbrev": "tbruTREU927", + "annotation_version": "Jul 10, 2018" }, { - "annotation_source": "GeneDB", "public_abbrev": "tbruTREU927", - "genome_version": "Jul 10, 2018", "annotation_version": "Jul 10, 2018", "genome_source": "GeneDB", + "annotation_source": "GeneDB", + "genome_version": "Jul 10, 2018", "build_number": "42" }, { - "annotation_version": "Jul 10, 2018", - "genome_source": "GeneDB", "build_number": "41", + "genome_source": "GeneDB", "genome_version": "Jul 10, 2018", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "annotation_source": "GeneDB", + "annotation_version": "Jul 10, 2018", + "public_abbrev": "tbruTREU927" }, { "public_abbrev": "tbruTREU927", - "genome_version": "Jul 10, 2018", - "build_number": "39", - "genome_source": "GeneDB", "annotation_version": "Jul 10, 2018", - "annotation_source": "GeneDB" - }, - { - "annotation_source": "GeneDB", - "annotation_version": "Jan 09, 2017", "genome_source": "GeneDB", - "build_number": "38", - "genome_version": "Jan 09, 2017", - "public_abbrev": "tbruTREU927" + "annotation_source": "GeneDB", + "genome_version": "Jul 10, 2018", + "build_number": "39" }, { "public_abbrev": "tbruTREU927", - "genome_version": "Jan 09, 2017", - "genome_source": "GeneDB", "annotation_version": "Jan 09, 2017", - "build_number": "37", - "annotation_source": "GeneDB" - }, - { "annotation_source": "GeneDB", - "public_abbrev": "tbruTREU927", "genome_version": "Jan 09, 2017", - "build_number": "36", "genome_source": "GeneDB", - "annotation_version": "Jan 09, 2017" + "build_number": "38" }, { + "build_number": "37", + "genome_source": "GeneDB", "annotation_source": "GeneDB", "genome_version": "Jan 09, 2017", - "build_number": "34", "annotation_version": "Jan 09, 2017", - "genome_source": "GeneDB", "public_abbrev": "tbruTREU927" }, { - "annotation_source": "GeneDB", "annotation_version": "Jan 09, 2017", + "public_abbrev": "tbruTREU927", + "build_number": "36", "genome_source": "GeneDB", - "build_number": "33", "genome_version": "Jan 09, 2017", - "public_abbrev": "tbruTREU927" + "annotation_source": "GeneDB" }, { - "annotation_source": "GeneDB", + "annotation_version": "Jan 09, 2017", "public_abbrev": "tbruTREU927", + "build_number": "34", "genome_source": "GeneDB", + "genome_version": "Jan 09, 2017", + "annotation_source": "GeneDB" + }, + { "annotation_version": "Jan 09, 2017", - "build_number": "32", + "public_abbrev": "tbruTREU927", + "build_number": "33", + "genome_source": "GeneDB", + "annotation_source": "GeneDB", "genome_version": "Jan 09, 2017" }, { + "build_number": "32", + "genome_source": "GeneDB", + "annotation_source": "GeneDB", "genome_version": "Jan 09, 2017", "annotation_version": "Jan 09, 2017", - "genome_source": "GeneDB", + "public_abbrev": "tbruTREU927" + }, + { "build_number": "31", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "genome_source": "GeneDB", + "genome_version": "Jan 09, 2017", + "annotation_source": "GeneDB", + "annotation_version": "Jan 09, 2017", + "public_abbrev": "tbruTREU927" }, { "annotation_source": "GeneDB", "genome_version": "May 19, 2016", - "annotation_version": "May 19, 2016", "genome_source": "GeneDB", "build_number": "29", - "public_abbrev": "tbruTREU927" + "public_abbrev": "tbruTREU927", + "annotation_version": "May 19, 2016" }, { - "build_number": "27", + "public_abbrev": "tbruTREU927", "annotation_version": "Aug 28, 2013", - "genome_source": "GeneDB", + "annotation_source": "GeneDB", "genome_version": "Aug 28, 2013", - "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "genome_source": "GeneDB", + "build_number": "27" }, { - "genome_version": "Aug 28, 2013", - "genome_source": "GeneDB", "annotation_version": "Aug 28, 2013", - "build_number": "20", "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "build_number": "20", + "genome_version": "Aug 28, 2013", + "annotation_source": "GeneDB", + "genome_source": "GeneDB" }, { - "annotation_source": "GeneDB", - "public_abbrev": "tbruTREU927", - "genome_source": "GeneDB", "annotation_version": "Jan 16, 2013", + "public_abbrev": "tbruTREU927", "build_number": "18", - "genome_version": "Jan 16, 2013" + "annotation_source": "GeneDB", + "genome_version": "Jan 16, 2013", + "genome_source": "GeneDB" }, { - "genome_version": "Oct 20, 2010", - "build_number": "0", - "genome_source": "GeneDB", - "annotation_version": "Oct 20, 2010", "public_abbrev": "tbruTREU927", - "annotation_source": "GeneDB" + "annotation_version": "Oct 20, 2010", + "genome_source": "GeneDB", + "annotation_source": "GeneDB", + "genome_version": "Oct 20, 2010", + "build_number": "0" } ] }, { - "name": "Trichoderma reesei QM6a", + "species_ncbi_tax_id": "9606", "is_reference_strain": "1", + "name": "Homo sapiens REF", + "internal_abbrev": "hsapREF", + "strain_abbrev": "REF", "is_annotated_genome": "1", + "name_for_filenames": "HsapiensREF", + "organism_abbrev": "hsapREF", "history": [ { - "annotation_version": "Nov 01, 2021", - "genome_version": "GCA_000167675.2", - "build_number": "56", - "public_abbrev": "treeQM6a", - "genome_source": "GenBank", - "annotation_source": "Apollo" - }, - { - "annotation_version": "Dec 13, 2012", - "genome_version": "GCA_000167675.2", - "build_number": "51", - "public_abbrev": "treeQM6a", - "genome_source": "GenBank", - "annotation_source": "GenBank" + "genome_source": "Ensembl", + "annotation_source": "Ensembl", + "genome_version": "GCA_000001405.28", + "build_number": "52", + "public_abbrev": "hsapREF", + "annotation_version": "GRCh38.p13" }, { - "annotation_source": "GenBank", - "genome_source": "GenBank", - "public_abbrev": "treeQM6a", "build_number": "29", - "genome_version": "Mar 14, 2015", - "annotation_version": "Dec 13, 2012" + "genome_source": "GENCODE", + "annotation_source": "GENCODE", + "genome_version": "GRCh38", + "annotation_version": "24", + "public_abbrev": "hsapREF" }, { - "annotation_source": "JGI", - "genome_source": "JGI", - "build_number": "19", - "public_abbrev": "treeQM6a", - "annotation_version": "2", - "genome_version": "2" + "build_number": "18", + "genome_source": "UCSC", + "genome_version": "hg19", + "annotation_source": "GENCODE", + "annotation_version": "13", + "public_abbrev": "hsapREF" } - ], - "name_for_filenames": "TreeseiQM6a", - "strain_abbrev": "QM6a", - "organism_abbrev": "treeQM6a", - "species_ncbi_tax_id": "51453" + ] } ] } \ No newline at end of file diff --git a/Model/t/portal.t b/Model/t/portal.t index 732e3e1c9f..73d7535e55 100644 --- a/Model/t/portal.t +++ b/Model/t/portal.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 19; +use Test::More tests => 22; use lib $ENV{GUS_HOME} . "/lib/perl"; use ApiCommonModel::Model::ApolloRelease::Portal; @@ -35,6 +35,25 @@ my $b = $orgs->{tbruLister427_2018}; is($b->{is_reference}, 0, 'non-reference strain is 0'); ok(defined $t->{latest_annotation_version}, 'latest annotation version derived'); + +# apidb.organism carries TWO identifiers and the jbrowse scripts disagree about +# which they take: jbrowseTracks selects `where public_abbrev = ?`, while the +# five track producers key `auto_generated//` off the INTERNAL one. +# cdenJEC21 is the only organism in the fixture where they differ -- it is the +# renamed Cryptococcus -- so it is the only case that can catch a caller +# feeding the wrong identifier. Feeding the public abbrev to a track producer +# gives exit 2 and a missing datasetAndPresenterProps.conf, and feeding the +# internal abbrev to jbrowseTracks silently matches nothing. +# +# `abbrev` stays PUBLIC deliberately: the hash key and every downstream +# consumer -- the Apollo roster, the overlay file, reconciliation -- already +# key on the public abbrev, which is also what the portal shows. The internal +# one rides alongside so a caller can resolve public -> internal once. +my $renamed = $orgs->{cdenJEC21}; +ok($renamed, 'the renamed organism is keyed by its PUBLIC abbrev'); +is($renamed->{abbrev}, 'cdenJEC21', 'abbrev remains the public abbrev, unchanged'); +is($renamed->{internal_abbrev}, 'cneoJEC21', + 'internal_abbrev carries apidb.organism.abbrev, which differs here'); is($P->qualifies($b), 0, 'non-reference organism does not qualify'); # "0.0" is truthy as a Perl string but zero numerically. Pin the coercion so a From 1b93b695c49e152baf4ab4dbec9438293a63e96f Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 21:57:18 -0400 Subject: [PATCH 27/42] Apollo spec: detect renames from the database, not the genome Measured: 37 of 831 organisms have an internal abbrev differing from their public one, and of Apollo's 459 directories, 457 name a public abbrev, 2 name an internal abbrev whose public has changed, and 0 name neither. Both live renames therefore fall out of the database exactly, with no file I/O. Assembly identity stays as the fallback for an orphan the database cannot explain -- an organism whose internal abbrev also moved. The Rename module is unchanged; the caller resolves the database renames first and passes only the residue to it. Also recorded: two case-insensitive collisions exist in the abbrev namespace (scerS288C/scerS288c, and one under bnonp57), so every comparison must be case-sensitive. A lc() here merges two organisms. Co-Authored-By: Claude Opus 5 --- ...026-08-21-apollo-release-package-design.md | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 75cc583246..d0c78a68e9 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -46,6 +46,8 @@ Every row measured on 2026-08-21, not inferred. | 15 | 17 live organisms are deliberately hidden, and the update command would un-hide them | `publicMode=false` on 17 organisms; 16 are on the portal and 15 qualify as reference+annotated, so they are curator-hidden, not retired. 3 carry annotations (`iscaPalLabHiFi` 20, `treeQM6a` 11, `etenHoughton2021` 2). Paul's `updateOrganismInfo` curl hardcodes `"publicMode":"true"`, so running the generated commands re-publishes all 17. | | 16 | `jbrowseRefSeqs` is dead for every organism | Exit 255, 211 bytes on stderr: `Can't locate object method "getCacheFile"`. Commit `e0e9a61bb` commented out the `getCacheFile`/`setCacheFile` accessors in `JBrowseUtil.pm` but left `printFromCache()` and `setCacheFileName()` calling them. Organism-independent; dies before any DB work. Served through `responseFromCommand`, so it corrupts the response body, not just a log. | | 17 | The two script families disagree about which organism abbrev they take | `apidb.organism` holds `abbrev = cneoJEC21`, `public_abbrev = cdenJEC21`. `jbrowseTracks` consumes the **public** abbrev and returns the internal one; the five track producers consume the **internal** abbrev and key `auto_generated//` off it. Feeding the wrong one gives exit 2, or a wrong-organism answer. | +| 18 | The internal/public split affects 37 organisms, and one pair differs only by case | Measured over all 831: `internal_abbrev ne organism_abbrev` for **37**, never undef. Includes cross-genus renames (`cglaCBS138`→`nglaCBS138`, `pory70-15`→`mory70-15`, `cgatVGIIR265`→`cdeuR265`) and `scerS288C`→`scerS288c`, which differs **only by case**. Two case-insensitive collisions exist across the whole namespace (`scers288c`, `bnonp57`), so every abbrev comparison must be case-sensitive. There are **no** exact collisions: no abbrev is one organism's public and another's internal, and no internal abbrev repeats. | +| 19 | Both live renames are detectable from the database alone | Of Apollo's 459 directories, 457 match a public abbrev, **2 match an internal abbrev whose public differs** (`cglaCBS138`, `cneoJEC21`), and 0 match neither. So a rename is "Apollo's directory names an internal abbrev whose organism now publishes under a different name" — exact, cheap, and authoritative. | ### Set sizes as of build 71 @@ -144,13 +146,29 @@ non-qualifying rather than letting it persist unnoticed. ### Rename detection -For each Apollo organism whose abbrev is absent from the portal, compare the `.fa.fai` -shipped in the previous release against the `.fai` of each portal organism sharing its -`strain_abbrev`. An identical set of sequence names and lengths means the same assembly -under a new name. +Two mechanisms, in order. The first is authoritative and handles every case seen so far; +the second is the fallback for cases the database cannot explain. -Species taxon ID is deliberately **not** used (fact 12). Strain abbrev narrows the -candidates; sequence identity decides. +**1. Internal abbrev (primary).** `apidb.organism` carries both `abbrev` (internal, stable) +and `public_abbrev` (what the portal shows). A taxonomic reclassification changes the public +one and leaves the internal one alone. Apollo's `directory` holds the public abbrev as of the +release that wrote it — so an orphan whose directory names an **internal** abbrev belongs to +the organism now publishing under a different name. Exact, no file I/O, and it resolved both +live cases (fact 19). + +Comparisons are **case-sensitive, always**. `scerS288C` and `scerS288c` are different +organisms (fact 18); a `lc()` anywhere in this path merges them. + +**2. Assembly identity (fallback).** For an orphan the database cannot explain — an organism +whose internal abbrev also changed, e.g. after a reload — compare the `.fa.fai` shipped in the +previous release against the `.fai` of each portal organism sharing its `strain_abbrev`. +Identical sequence names and lengths mean the same assembly under a new name. + +Species taxon ID is deliberately **not** used by either mechanism (fact 12). + +Strain abbrev narrows the candidates; sequence identity decides. An ambiguous match yields no +rename: guessing repoints curated annotations onto the wrong genome, which is worse than +leaving a prune candidate for a human. A rename emits **one** update against the existing Apollo `id`, setting `directory`, `blatdb`, and `commonName`. Emitting an add for the new abbrev in the same run is a hard From d026a81f42755bf3a00a152d8a0c49e5f3d0062e Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 22:25:07 -0400 Subject: [PATCH 28/42] Apollo: derive refSeqs.json from the .fai; jbrowseRefSeqs is orphaned Both JBrowseService endpoints that called jbrowseRefSeqs are commented out, with the note "THIS SHOULD BE REPLACED BY INDEXED FASTA IN WEBSERVICES", and nothing else references the script. It was already dead when it broke -- which is why nobody noticed for six months. Corrects an earlier claim in this branch that the breakage was corrupting the live site's response payload. It was not; the endpoint serves nothing. The package does not need it either: release-68's trackList already pointed refSeqs at seq/.fa.fai with an IndexedFasta store, and the refSeqs.json in seq/ is referenced by nothing. The tool now derives it from the .fai it already copies -- same shape, no database round trip, no dependency on a dead script. Co-Authored-By: Claude Opus 5 --- .../2026-08-21-apollo-release-package.md | 9 ++++--- ...026-08-21-apollo-release-package-design.md | 27 ++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/plans/2026-08-21-apollo-release-package.md b/docs/superpowers/plans/2026-08-21-apollo-release-package.md index 9b75129992..be06a2d530 100644 --- a/docs/superpowers/plans/2026-08-21-apollo-release-package.md +++ b/docs/superpowers/plans/2026-08-21-apollo-release-package.md @@ -2221,10 +2221,13 @@ sub generateOrganism { copyConf("$ENV{GUS_HOME}/lib/jbrowse/functions.conf", "$orgDir/functions.conf", $base, 0); - my $refSeqs = runCapture("$ENV{GUS_HOME}/bin/jbrowseRefSeqs", - $ENV{GUS_HOME}, $project, $abbrev); + # Derived from the .fai we just copied. jbrowseRefSeqs is orphaned -- both + # service endpoints that called it are commented out -- and the package has + # never referenced refSeqs.json anyway; trackList points at the .fai. + my $refSeqs = ApiCommonModel::Model::ApolloRelease::Generate->refSeqsFromFai( + "$seqDir/$abbrev.fa.fai"); ApiCommonModel::Model::ApolloRelease::Generate->writeFile( - "$seqDir/refSeqs.json", $refSeqs, $base); + "$seqDir/refSeqs.json", JSON::encode_json($refSeqs), $base); my $trackListJson = runCapture("$ENV{GUS_HOME}/bin/jbrowseTracks", $abbrev, $project, 0, 'geneAnnotationTracks'); diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index d0c78a68e9..20308112f9 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -192,9 +192,34 @@ Per organism in the approved roster: | `organismSpecific.json` | `jbrowseOrganismSpecificTracks UniDB 1 jbrowse` | | `dnaseq.json` | `jbrowseDNASeqTracks UniDB jbrowse` | | `seq/.fa`, `seq/.fa.fai` | copied and renamed from webServices | -| `seq/refSeqs.json` | `jbrowseRefSeqs $GUS_HOME UniDB ` | +| `seq/refSeqs.json` | derived from the copied `.fai` — one `{name, start: 0, end: length, length}` per sequence. **Not** `jbrowseRefSeqs`; see below. | | `twoBit/.2bit` | `faToTwoBit` — the only new computation | +### refSeqs.json is derived, not queried + +`jbrowseRefSeqs` is **orphaned**. Both service endpoints that called it are commented out in +`JBrowseService.java` with the note *"THIS SHOULD BE REPLACED BY INDEXED FASTA IN +WEBSERVICES"*, and nothing else in any repo references it. That is why fact 16 went unnoticed +for six months: the script was already dead when it broke. + +The package does not need it either. Release-68's own `trackList.json` already pointed +`refSeqs` at `seq/.fa.fai` and used an `IndexedFasta` store with a `faiUrlTemplate`; +the `refSeqs.json` sitting in `seq/` is referenced by nothing — not by `refSeqs`, not in the +`include` list. + +So the tool derives it from the `.fai` it already copies, rather than resurrecting a dead +DB-backed script: + +``` +IV 1005040 ... -> {"name": "IV", "start": 0, "end": 1005040, "length": 1005040} +``` + +Kept rather than dropped because Apollo's pre-release checklist lists it and nothing here can +prove Apollo ignores it; deriving it costs one pass over a file we already read, removes a +per-organism database round-trip, and preserves the b68 output shape. One difference worth +recording: entry order follows the fasta index rather than the old query's +chromosome-then-length ordering. Nothing observed reads this file, let alone its order. + ### faToTwoBit on cedar `faToTwoBit` existed only on yew, at `/eupath/workflow-software/bin/faToTwoBit` — a 2016 From f80cfb019f3c822f99745f5b2c5686043846c0a2 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 22:30:52 -0400 Subject: [PATCH 29/42] Apollo spec: Apollo owns its fasta copy; organismSpecific needs a strip and has junk 20. Apollo's Organism record points at seq/.fa relative to its own data dir and derives its sequences count from that index, while genomic sites fetch the reference through the store service. The store location genuinely differs, so the package must ship its own copy. 21. The refseqs track is no longer in tracks.conf -- it is in organismSpecific.json, pointing at the store URL, and must be stripped there instead or Apollo ends up with two reference tracks. While measuring that: addChipChipTracks pushes the return value of makeChipChipPeak/makeChipChipSmoothed, which already push and so return the new array length. 56 bare integers among 158 track entries for tgonME49, 28 of 103 for pfal3D7. That endpoint is live. Co-Authored-By: Claude Opus 5 --- .../specs/2026-08-21-apollo-release-package-design.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 20308112f9..3410bcf0db 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -48,6 +48,8 @@ Every row measured on 2026-08-21, not inferred. | 17 | The two script families disagree about which organism abbrev they take | `apidb.organism` holds `abbrev = cneoJEC21`, `public_abbrev = cdenJEC21`. `jbrowseTracks` consumes the **public** abbrev and returns the internal one; the five track producers consume the **internal** abbrev and key `auto_generated//` off it. Feeding the wrong one gives exit 2, or a wrong-organism answer. | | 18 | The internal/public split affects 37 organisms, and one pair differs only by case | Measured over all 831: `internal_abbrev ne organism_abbrev` for **37**, never undef. Includes cross-genus renames (`cglaCBS138`→`nglaCBS138`, `pory70-15`→`mory70-15`, `cgatVGIIR265`→`cdeuR265`) and `scerS288C`→`scerS288c`, which differs **only by case**. Two case-insensitive collisions exist across the whole namespace (`scers288c`, `bnonp57`), so every abbrev comparison must be case-sensitive. There are **no** exact collisions: no abbrev is one organism's public and another's internal, and no internal abbrev repeats. | | 19 | Both live renames are detectable from the database alone | Of Apollo's 459 directories, 457 match a public abbrev, **2 match an internal abbrev whose public differs** (`cglaCBS138`, `cneoJEC21`), and 0 match neither. So a rename is "Apollo's directory names an internal abbrev whose organism now publishes under a different name" — exact, cheap, and authoritative. | +| 20 | Apollo's Organism record references the fasta itself, by a path relative to its data dir | All 459 carry `genomeFasta = seq/.fa`, `genomeFastaIndex = seq/.fa.fai`, and a non-zero `sequences` count derived from that index. Genomic sites instead fetch the reference through `/a/service/jbrowse/store?data=/genomeAndProteome/fasta/genome.fasta`. **The store location differs**, so the package must ship its own copy and keep those paths relative. | +| 21 | The reference track moved from `tracks.conf` into `organismSpecific.json`, and that file also contains junk | Every organism's `organismSpecific.json` carries one `label: refseqs` `IndexedFasta` track with `useAsRefSeqStore`, pointing at the **store URL** — it must be stripped, or Apollo gets two reference tracks, one remote. Separately, `addChipChipTracks` appends the return value of `makeChipChipPeak`/`makeChipChipSmoothed`, which already push and therefore return the new array length: **56 bare integers among 158 entries for `tgonME49`, 28 of 103 for `pfal3D7`**, 0 for an organism with no ChIP-chip data. This endpoint is live, so the site serves it too. | ### Set sizes as of build 71 From b5434f5de9854d0d78fa843ac48b4af83746f09e Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 22:45:45 -0400 Subject: [PATCH 30/42] feat(apollo): generate the per-organism Apollo package Resolves the public/internal abbrev split -- the track scripts take the internal abbrev, output paths and Apollo directories use the public one. Strips the store-URL reference track from organismSpecific.json so Apollo uses its own local copy, and derives refSeqs.json from the .fai rather than the orphaned jbrowseRefSeqs. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Generate.pm | 571 +++++++++++++++++++++++ Model/t/generate.t | 191 ++++++++ 2 files changed, 762 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Generate.pm create mode 100644 Model/t/generate.t diff --git a/Model/lib/perl/ApolloRelease/Generate.pm b/Model/lib/perl/ApolloRelease/Generate.pm new file mode 100644 index 0000000000..918c705aa2 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Generate.pm @@ -0,0 +1,571 @@ +package ApiCommonModel::Model::ApolloRelease::Generate; + +use strict; +use warnings; + +use JSON; +use File::Path qw(make_path); +use File::Copy qw(cp); +use File::Temp; + +use ApiCommonModel::Model::ApolloRelease::Absolutize; + +my $ABS = 'ApiCommonModel::Model::ApolloRelease::Absolutize'; + +# canonical() so two runs over the same model produce byte-identical files. +# The release is diffed against the previous one by hand; hash order noise +# would bury the handful of real changes in a few hundred thousand lines. +# +# utf8() so encode() returns OCTETS, not characters. decode_json hands back +# decoded characters, and several organisms carry non-ASCII in a track key or +# description (tgonME49 does), so encoding without this produces a character +# string that `print` emits with a "Wide character" warning. The bytes on disk +# would have been right; the warning is the problem -- this module's own policy +# (_defaultRunScript, Portal.pm) is that any stderr byte from a config producer +# means its output is untrustworthy, and these files are also served through +# responseFromCommand, which splices stderr into the JSON body. +my $JSON = JSON->new->canonical->utf8; + +# Exposed so a test can assert the encoder really emits octets. Everything +# this module writes goes through it. +sub encodeJson { + my ($class, $document) = @_; + return $JSON->encode($document); +} + +# Turns one portal organism into the directory Apollo reads. +# +# TWO ABBREVS, and mixing them up is the single easiest mistake here. See +# Portal.pm: `abbrev` is public_abbrev, `internal_abbrev` is apidb.organism.abbrev, +# and they differ for 37 of 831 organisms. +# - the five track-producing scripts take the INTERNAL abbrev, because they +# key auto_generated// off it directly (a wrong one is exit 2 on a +# missing datasetAndPresenterProps.conf); +# - jbrowseTracks takes the PUBLIC abbrev (its SQL is `where public_abbrev = ?`); +# - every output path, and Apollo's own data directory, uses the PUBLIC abbrev, +# because that is the identity Apollo already holds for the organism. +# Comparisons are case sensitive throughout: scerS288C and scerS288c are two +# different organisms. +# +# Everything that produces text is split into a pure function and a seam, so +# the rules can be exercised with no database, no GUS_HOME and no filesystem. +# The seams are coderefs in %opts, in the style Rename.pm uses for its .fai +# lookups. + +# Warnings are COLLECTED, never written to stderr -- same policy as Portal.pm +# and Rename.pm. These scripts are also served through responseFromCommand, +# which merges stderr into the JSON body, so this module treats a child's +# stderr as proof its output is untrustworthy; it must not then emit stderr of +# its own for a recoverable skip. +my @WARNINGS; + +sub warnings { return @WARNINGS } + +# --------------------------------------------------------------------------- +# PURE: include URL -> the local filename Apollo will read it as +# --------------------------------------------------------------------------- + +# Ordered: the first match wins. rnaseqJunctions MUST precede rnaseq -- both +# endpoints are generated for the same organism, so the wrong order writes one +# file's contents under the other's name, producing a package that parses, and +# loads, and shows the wrong tracks. +my @INCLUDE_NAMES = ( + [qr{apollo_gene_tracks\.conf$} => 'apollo_gene_tracks.conf'], + [qr{functions\.conf$} => 'functions.conf'], + [qr{bindingSites\.conf$} => 'bindingSites.conf'], + [qr{tracks\.conf$} => 'tracks.conf'], + [qr{rnaseqJunctions}i => 'rnaseqJunctions.json'], + [qr{organismSpecific}i => 'organismSpecific.json'], + [qr{rnaseq}i => 'rnaseq.json'], + [qr{chipseq}i => 'chipseq.json'], + [qr{dnaseq}i => 'dnaseq.json'], +); + +# Includes that are deliberately NOT carried into the package. +# user-datasets-jbrowse: per-user data behind a session cookie. Apollo has no +# session, so the include would 404 on every load. +# jbrowse_embed.conf: styling for JBrowse embedded in a VEuPathDB gene page. +# Apollo embeds JBrowse itself and supplies its own chrome; release-68 never +# carried it, because Paul's script read the DEFAULT jbrowseTracks track set, +# which does not name it. +my @DROP_INCLUDES = (qr{user-datasets-jbrowse}, qr{jbrowse_embed\.conf$}); + +sub localNameForInclude { + my ($class, $url) = @_; + + foreach my $rule (@INCLUDE_NAMES) { + my ($pattern, $name) = @$rule; + return $name if $url =~ $pattern; + } + + return undef; +} + +# --------------------------------------------------------------------------- +# PURE: the site's trackList.json -> Apollo's +# --------------------------------------------------------------------------- + +# Three deliberate differences from what the site serves, all declared in spec +# section 10 so that a diff against the live site can be audited: +# 1. include URLs rewritten to the local filenames (and the drops above) +# 2. refSeqs pointed at the local .fai +# 3. tracks replaced with a single local IndexedFasta reference track +# +# $extraIncludes is the set of files the CALLER actually wrote and that the +# skeleton does not name -- functions.conf and apollo_gene_tracks.conf are +# pushed on by this tool, not by jbrowseTracks, and with the geneAnnotationTracks +# skeleton so are rnaseq/chipseq/dnaseq. Deriving the include list from what +# was really produced is what makes "every include resolves to a file that +# exists and parses" true by construction rather than by inspection. +sub buildTrackList { + my ($class, $trackList, $abbrev, $base, $extraIncludes) = @_; + + # Shallow copy, then replace every nested structure we touch. generateAll + # runs this once per organism and a shared nested arrayref would accumulate + # the previous organism's includes. + my %built = %$trackList; + + my @includes; + my %seen; + + URL: foreach my $url (@{$trackList->{include} || []}) { + foreach my $drop (@DROP_INCLUDES) { + next URL if $url =~ $drop; + } + + my $name = $class->localNameForInclude($url); + + # Not fatal, but never silent. An unmapped include means jbrowseTracks + # grew an endpoint this tool does not know how to fetch; the package is + # still usable, and a human needs to decide whether Apollo wants it. + unless ($name) { + push @WARNINGS, "$abbrev: unrecognised include '$url'; not carried into the package"; + next URL; + } + + next if $seen{$name}++; + push @includes, $name; + } + + foreach my $name (@{$extraIncludes || []}) { + next if $seen{$name}++; + push @includes, $name; + } + + $built{include} = \@includes; + + # Apollo owns its own copy of the genome; the site's refSeqs pointed at a + # store URL. + $built{refSeqs} = "seq/$abbrev.fa.fai"; + + $built{names} = {%{$trackList->{names} || {}}}; + $built{names}{url} = $ABS->rewrite($built{names}{url}, $base) + if defined $built{names}{url}; + + $built{tracks} = [ + { + category => "Sequence Analysis", + faiUrlTemplate => "seq/$abbrev.fa.fai", + key => "Reference sequence", + label => "DNA", + seqType => "dna", + storeClass => "JBrowse/Store/SeqFeature/IndexedFasta", + type => "SequenceTrack", + urlTemplate => "seq/$abbrev.fa", + useAsRefSeqStore => JSON::true, + } + ]; + + return \%built; +} + +# --------------------------------------------------------------------------- +# PURE: refSeqs.json derived from the .fai +# --------------------------------------------------------------------------- + +# NOT from jbrowseRefSeqs, which is orphaned: both service endpoints that +# called it are commented out, and it has died with a missing getCacheFile for +# every organism since commit e0e9a61bb. The .fai we already copy carries the +# same two facts, so this is one extra pass over a file in hand instead of a +# per-organism database round trip. +# +# Dies rather than warns on a malformed line. Rename.pm can afford to shrug at +# a bad index (the organism just stays a prune candidate); here the output IS +# the index, and half of one is a package that loads with sequences missing. +sub refSeqsFromFai { + my ($class, $text) = @_; + + my @refSeqs; + my $lineNumber = 0; + + foreach my $line (split /\n/, ($text // '')) { + $lineNumber++; + next unless length $line; + + my ($name, $length) = split /\t/, $line; + + die "malformed .fai line $lineNumber: '$line'\n" + unless defined $name && length $name && defined $length && $length =~ /^\d+$/; + + # +0 so these encode as JSON numbers. JBrowse compares them + # arithmetically, and a quoted "1876705" sorts as a string. + push @refSeqs, {name => $name, start => 0, end => $length + 0, length => $length + 0}; + } + + return \@refSeqs; +} + +# --------------------------------------------------------------------------- +# PURE: strip the store-URL reference track +# --------------------------------------------------------------------------- + +# Every organism's organismSpecific.json carries one IndexedFasta track with +# useAsRefSeqStore true, pointing at the site's store URL. Apollo must use the +# local copy this tool writes into seq/ instead; two tracks both claiming to be +# the reference sequence store is not a merge JBrowse resolves in our favour. +# +# Filter on the FIELD, never on the label. The label ("refseqs") is cosmetic: +# one organism's presenter renaming it would leave the store-URL track in place +# with nothing failing. +sub stripRefSeqStoreTracks { + my ($class, $document) = @_; + + my %copy = %$document; + my @kept; + my $removed = 0; + + foreach my $track (@{$document->{tracks} || []}) { + if (ref $track eq 'HASH' && $track->{useAsRefSeqStore}) { + $removed++; + next; + } + push @kept, $track; + } + + $copy{tracks} = \@kept; + + return (\%copy, $removed); +} + +# --------------------------------------------------------------------------- +# PURE: invariants over a generated tracks array +# --------------------------------------------------------------------------- + +# A bug fixed on master today had addChipChipTracks pushing bare integers onto +# the tracks array -- 56 of 158 entries for tgonME49. The JSON stayed valid, +# so nothing upstream of a curator's browser noticed. This module is the last +# gate before the package ships, and the check costs one pass. +sub assertTracksAreObjects { + my ($class, $document, $label) = @_; + + die "$label: expected a 'tracks' array\n" + unless ref($document->{tracks} || []) eq 'ARRAY'; + + my @bad; + my $tracks = $document->{tracks} || []; + + for (my $i = 0; $i < @$tracks; $i++) { + push @bad, $i unless ref $tracks->[$i] eq 'HASH'; + } + + return 1 unless @bad; + + my @shown = @bad > 5 ? @bad[0 .. 4] : @bad; + die "$label: " . scalar(@bad) . " of " . scalar(@$tracks) + . " entries in 'tracks' are not JSON objects, at index(es) " + . join(', ', @shown) . (@bad > @shown ? ', ...' : '') . "\n"; +} + +# Zero is a COUNT, not a failure. cneoJEC21 genuinely has 0 ChIP-Seq and 0 +# DNA-Seq tracks: its datasetAndPresenterProps.conf carries the template +# anchors with no injected entries beneath them, because no such data is +# loaded. Failing on it would block a correct release; hiding it would let the +# flat-file migration's characteristic empty-but-valid output through unseen. +# So the caller records it and a human judges. +sub countTracks { + my ($class, $document) = @_; + return scalar @{$document->{tracks} || []}; +} + +# --------------------------------------------------------------------------- +# IO: write one file, absolutized, with the post-condition enforced +# --------------------------------------------------------------------------- + +sub writeFile { + my ($class, $path, $content, $base) = @_; + + my $rewritten = $ABS->rewrite($content, $base); + + # Not optional. The previous script did the same rewrite with no check, so a + # missed URL became a track that 404s inside Apollo: the config loads, the + # track appears, and it is empty. + $ABS->assertNoRelative($rewritten, $path); + + # ':raw' deliberately: every producer above hands this octets already (the + # JSON encoder is utf8(), the .conf files are slurped as bytes), so any + # encoding layer here would be a second, wrong one. + open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; + print $fh $rewritten; + close $fh or die "Cannot close $path: $!\n"; + + return 1; +} + +# --------------------------------------------------------------------------- +# The seams: everything that shells out or touches disk +# --------------------------------------------------------------------------- + +# Runs one of the jbrowse* producers and returns its stdout. +# +# ANY byte on stderr is fatal, matching Portal.pm. These scripts are also +# served through responseFromCommand, which merges stderr into the response +# body, so a stray Perl warning is a correctness bug rather than a log line -- +# and the JSON we are about to parse may be the warning text spliced into it. +sub _defaultRunScript { + my ($gusHome, $script, @args) = @_; + + my $errFile = File::Temp->new(TEMPLATE => 'apolloRelease.XXXXXXXX', + TMPDIR => 1, SUFFIX => '.err'); + + my $quoted = join ' ', map { "'" . (s/'/'\\''/gr) . "'" } @args; + my $cmd = "$gusHome/bin/$script $quoted"; + + my $out = `$cmd 2>'$errFile'`; + + if ($?) { + $errFile->unlink_on_destroy(0); + die "$script failed (exit " . ($? >> 8) . "); see $errFile\n"; + } + + my $errSize = -s "$errFile" || 0; + if ($errSize) { + $errFile->unlink_on_destroy(0); + die "$script wrote $errSize bytes to stderr; refusing to trust its output.\n" + . "See $errFile\n"; + } + + return $out; +} + +sub _defaultReadFile { + my ($path) = @_; + open(my $fh, '<', $path) or die "Cannot read $path: $!\n"; + local $/; + my $content = <$fh>; + close $fh; + return $content; +} + +sub _defaultCopyFile { + my ($from, $to) = @_; + cp($from, $to) or die "Cannot copy $from to $to: $!\n"; + return 1; +} + +sub _defaultTwoBit { + my ($fasta, $out) = @_; + system('faToTwoBit', $fasta, $out) == 0 + or die "faToTwoBit $fasta $out failed: $?\n"; + return 1; +} + +# Checked once, at startup, not 400 organisms into a run. +sub assertToolsAvailable { + my ($class) = @_; + + my $found = `which faToTwoBit 2>/dev/null`; + chomp $found; + + die "faToTwoBit is not on PATH.\n" + . "Install UCSC's current linux.x86_64 build to ~/bin/faToTwoBit.\n" + . "(The 2016 copy at /eupath/workflow-software/bin/faToTwoBit links against\n" + . " libssl.so.10 and does not run on cedar.)\n" + unless $found && -x $found; + + return $found; +} + +# --------------------------------------------------------------------------- +# One organism, end to end +# --------------------------------------------------------------------------- + +# The five track producers, each with the argv jbrowse* expects. Table driven +# because the argument ORDER differs per script for no reason anyone remembers, +# and inlining five near-identical call sites is how one of them ends up with +# the build number where the wsDir goes. +my @TRACK_SOURCES = ( + {file => 'rnaseq.json', script => 'jbrowseRnaAndChipSeqTracks', + args => sub { my ($i, $o) = @_; ($i, $o->{project}, $o->{build}, $o->{wsDir}, 'RNASeq', 'jbrowse') }}, + {file => 'chipseq.json', script => 'jbrowseRnaAndChipSeqTracks', + args => sub { my ($i, $o) = @_; ($i, $o->{project}, $o->{build}, $o->{wsDir}, 'ChIPSeq', 'jbrowse') }}, + {file => 'rnaseqJunctions.json', script => 'jbrowseRNASeqJunctionTracks', + args => sub { my ($i, $o) = @_; ($i, $o->{project}, $o->{build}, $o->{wsDir}, 1, 'jbrowse') }}, + {file => 'organismSpecific.json', script => 'jbrowseOrganismSpecificTracks', + args => sub { my ($i, $o) = @_; ($i, $o->{project}, 1, $o->{build}, $o->{wsDir}, 'jbrowse') }}, + {file => 'dnaseq.json', script => 'jbrowseDNASeqTracks', + args => sub { my ($i, $o) = @_; ($i, $o->{project}, $o->{build}, $o->{wsDir}, 'jbrowse') }}, +); + +# Static config copied verbatim (after absolutization) into every organism dir. +# +# apollo_gene_tracks.conf is NOT generated by anything: it is a checked-in file, +# ApiCommonModel/Model/lib/jbrowse/apollo_gene_tracks.conf, installed to +# $GUS_HOME/lib/jbrowse/ and separately to the webapp by ApiCommonWebsite's +# build.xml. It defines [tracks.processed_transcripts] -- category "Draggable +# Annotation" -- which is the track curators drag genes INTO. It is the reason +# Apollo exists, so it ships for every organism in the roster. Paul's "TODO: +# only include this for annotated genomes" is already satisfied upstream of +# here: Portal::qualifies admits only reference AND annotated genomes. +my @STATIC_CONFIGS = ( + {file => 'functions.conf', from => sub { "$_[0]/lib/jbrowse/functions.conf" }}, + {file => 'apollo_gene_tracks.conf', from => sub { "$_[0]/lib/jbrowse/apollo_gene_tracks.conf" }}, +); + +# $organism - a Portal.pm organism (needs abbrev, internal_abbrev, name_for_filenames) +# %$opts - outDir, base, project, build, wsDir, gusHome, plus optional seams: +# runScript($gusHome, $script, @args) -> stdout +# readFile($path) -> text +# copyFile($from, $to) +# twoBit($fasta, $out) +sub generateOrganism { + my ($class, $organism, $opts) = @_; + + my $abbrev = $organism->{abbrev} + or die "organism has no public abbrev\n"; + my $internal = $organism->{internal_abbrev} || $abbrev; + my $files = $organism->{name_for_filenames} + or die "$abbrev: no name_for_filenames; cannot locate its webServices genome\n"; + + my $runScript = $opts->{runScript} || \&_defaultRunScript; + my $readFile = $opts->{readFile} || \&_defaultReadFile; + my $copyFile = $opts->{copyFile} || \&_defaultCopyFile; + my $twoBit = $opts->{twoBit} || \&_defaultTwoBit; + + my $base = $opts->{base}; + my $gusHome = $opts->{gusHome}; + + my $organismDir = "$opts->{outDir}/data/$abbrev"; + my $seqDir = "$organismDir/seq"; + my $twoBitDir = "$opts->{outDir}/twoBit"; + make_path($seqDir, $twoBitDir); + + my %summary = (abbrev => $abbrev, internal_abbrev => $internal, + counts => {}, empty => [], stripped => 0); + + # --- the five generated track files ------------------------------------- + my @generated; + foreach my $source (@TRACK_SOURCES) { + my @args = $source->{args}->($internal, $opts); + my $json = $runScript->($gusHome, $source->{script}, @args); + + my $document = eval { decode_json($json) }; + die "$abbrev: $source->{script} did not return JSON for $source->{file}\n" + unless $document; + + $class->assertTracksAreObjects($document, "$abbrev/$source->{file}"); + + if ($source->{file} eq 'organismSpecific.json') { + my $removed; + ($document, $removed) = $class->stripRefSeqStoreTracks($document); + $summary{stripped} = $removed; + + # Exactly one is expected. Zero means the site stopped emitting it and + # this strip has quietly become a no-op; more than one means something + # changed shape. Neither is fatal -- Apollo's own local track is what + # the package uses either way -- but both want a human. + push @WARNINGS, "$abbrev: organismSpecific.json had $removed useAsRefSeqStore " + . "track(s); expected exactly 1" + unless $removed == 1; + } + + my $count = $class->countTracks($document); + $summary{counts}{$source->{file}} = $count; + push @{$summary{empty}}, $source->{file} unless $count; + + $class->writeFile("$organismDir/$source->{file}", $class->encodeJson($document), $base); + push @generated, $source->{file}; + } + + # --- tracks.conf, keyed off the INTERNAL abbrev ------------------------- + $class->writeFile("$organismDir/tracks.conf", + $readFile->("$gusHome/lib/jbrowse/auto_generated/$internal/tracks.conf"), + $base); + push @generated, 'tracks.conf'; + + # --- static config ------------------------------------------------------ + foreach my $static (@STATIC_CONFIGS) { + $class->writeFile("$organismDir/$static->{file}", + $readFile->($static->{from}->($gusHome)), $base); + push @generated, $static->{file}; + } + + # --- the genome, its index, and refSeqs.json ---------------------------- + # Copied VERBATIM, not through writeFile: this is sequence data, an + # absolutization pass over 67MB of nucleotides would find nothing and cost + # real time, and the .fai carries no self-reference so renaming it is safe. + my $fastaSource = "$opts->{wsDir}/$opts->{project}/build-$opts->{build}" + . "/$files/genomeAndProteome/fasta/genome.fasta"; + + $copyFile->($fastaSource, "$seqDir/$abbrev.fa"); + $copyFile->("$fastaSource.fai", "$seqDir/$abbrev.fa.fai"); + + my $refSeqs = $class->refSeqsFromFai($readFile->("$seqDir/$abbrev.fa.fai")); + die "$abbrev: the copied .fai describes no sequences\n" unless @$refSeqs; + + $class->writeFile("$seqDir/refSeqs.json", $class->encodeJson($refSeqs), $base); + $summary{sequences} = scalar @$refSeqs; + + # --- .2bit for BLAT ----------------------------------------------------- + # Always regenerated, never copied forward: 0.17-0.74s per genome, and an + # incremental scheme keyed on genome version adds state to save nothing. + $twoBit->("$seqDir/$abbrev.fa", "$twoBitDir/$abbrev.2bit"); + + # --- trackList.json, last, naming exactly what was written -------------- + # jbrowseTracks takes the PUBLIC abbrev. + my $skeleton = decode_json($runScript->($gusHome, 'jbrowseTracks', + $abbrev, $opts->{project}, 0, 'geneAnnotationTracks')); + + my $trackList = $class->buildTrackList($skeleton, $abbrev, $base, \@generated); + $class->writeFile("$organismDir/trackList.json", $class->encodeJson($trackList), $base); + + $summary{includes} = $trackList->{include}; + + return \%summary; +} + +# --------------------------------------------------------------------------- +# The whole roster, with failures isolated +# --------------------------------------------------------------------------- + +# Paul's script died on the first missing input, discarding hours of completed +# work. The organisms are independent and the run is hours long, so a failure +# is recorded and the loop continues; the caller exits non-zero on any entry in +# {failed}. +sub generateAll { + my ($class, $organisms, $perOrganism) = @_; + + @WARNINGS = (); + + my %results = (succeeded => [], failed => [], errors => {}, summaries => {}); + + foreach my $organism (@$organisms) { + my $abbrev = $organism->{abbrev}; + + my $summary = eval { $perOrganism->($organism) }; + + if (defined $summary) { + push @{$results{succeeded}}, $abbrev; + $results{summaries}{$abbrev} = $summary; + } + else { + my $error = $@ || "returned nothing and set no error"; + chomp $error; + push @{$results{failed}}, $abbrev; + $results{errors}{$abbrev} = $error; + } + } + + return \%results; +} + +1; diff --git a/Model/t/generate.t b/Model/t/generate.t new file mode 100644 index 0000000000..d9cc3394db --- /dev/null +++ b/Model/t/generate.t @@ -0,0 +1,191 @@ +use strict; +use warnings; +use Test::More; +use File::Temp qw(tempdir); +use JSON; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Generate; + +my $G = 'ApiCommonModel::Model::ApolloRelease::Generate'; +my $BASE = 'https://veupathdb.org'; + +# --------------------------------------------------------------------------- +# localNameForInclude +# --------------------------------------------------------------------------- + +is($G->localNameForInclude('/a/jbrowse/tracks/tgonME49/tracks.conf'), 'tracks.conf', + 'per-organism tracks.conf'); +is($G->localNameForInclude('/a/jbrowse/functions.conf'), 'functions.conf', + 'functions.conf'); +is($G->localNameForInclude('/a/jbrowse/apollo_gene_tracks.conf'), 'apollo_gene_tracks.conf', + 'apollo_gene_tracks.conf'); + +# rnaseqJunctions must not be swallowed by the plain rnaseq rule. Both endpoints +# are generated for the same organism, so a rule-order slip silently writes one +# file's contents under the other's name and nothing downstream notices. +is($G->localNameForInclude('/a/service/jbrowse/rnaseqJunctions/tgonME49'), + 'rnaseqJunctions.json', 'rnaseqJunctions wins over rnaseq'); +is($G->localNameForInclude('/a/service/jbrowse/rnaseq/tgonME49'), 'rnaseq.json', 'rnaseq'); +is($G->localNameForInclude('/a/service/jbrowse/chipseq/tgonME49'), 'chipseq.json', 'chipseq'); +is($G->localNameForInclude('/a/service/jbrowse/dnaseq/tgonME49'), 'dnaseq.json', 'dnaseq'); +is($G->localNameForInclude('/a/service/jbrowse/organismSpecific/tgonME49'), + 'organismSpecific.json', 'organismSpecific'); +is($G->localNameForInclude('/a/service/jbrowse/somethingNew/tgonME49'), undef, + 'an unrecognised include maps to undef rather than being guessed at'); + +# --------------------------------------------------------------------------- +# buildTrackList +# --------------------------------------------------------------------------- + +my $trackList = { + refSeqs => "/a/service/jbrowse/store?data=TgondiiME49/genomeAndProteome/fasta/genome.fasta.fai", + names => {type => "REST", url => "/a/service/jbrowse/names/tgonME49"}, + include => [ + "/a/jbrowse/tracks/tgonME49/tracks.conf", + "/a/service/jbrowse/rnaseqJunctions/tgonME49", + "/a/service/users/current/user-datasets-jbrowse/TgondiiME49", + "/a/jbrowse/jbrowse_embed.conf", + "/a/service/jbrowse/organismSpecific/tgonME49", + ], + tracks => [{label => "should be replaced"}], +}; + +my $built = $G->buildTrackList($trackList, 'tgonME49', $BASE); + +is_deeply([grep { /user-datasets/ } @{$built->{include}}], [], + 'user dataset includes are dropped'); +is_deeply([grep { /jbrowse_embed/ } @{$built->{include}}], [], + 'the site embed config is dropped'); +is_deeply($built->{include}, + ['tracks.conf', 'rnaseqJunctions.json', 'organismSpecific.json'], + 'includes rewritten to local filenames, in the order given'); +is($built->{refSeqs}, 'seq/tgonME49.fa.fai', 'refSeqs points at the local index'); +is(scalar(@{$built->{tracks}}), 1, 'exactly one track remains'); +is($built->{tracks}[0]{storeClass}, 'JBrowse/Store/SeqFeature/IndexedFasta', + 'and it is the local reference sequence track'); +is($built->{tracks}[0]{urlTemplate}, 'seq/tgonME49.fa', 'pointing at the local fasta'); +is($built->{tracks}[0]{faiUrlTemplate}, 'seq/tgonME49.fa.fai', 'and at the local index'); +like($built->{names}{url}, qr{^https://veupathdb\.org/a/}, 'names url absolutized'); + +# The caller owns the file set it actually produced, so it appends those names; +# buildTrackList must not invent them and must not duplicate one already mapped +# in from the skeleton. +my $withExtras = $G->buildTrackList($trackList, 'tgonME49', $BASE, + ['functions.conf', 'apollo_gene_tracks.conf', + 'rnaseq.json', 'organismSpecific.json']); +is_deeply($withExtras->{include}, + ['tracks.conf', 'rnaseqJunctions.json', 'organismSpecific.json', + 'functions.conf', 'apollo_gene_tracks.conf', 'rnaseq.json'], + 'extra local files are appended once, never duplicated'); + +# The input must not be mutated -- generateAll reuses one skeleton per organism +# and a shared nested arrayref would accumulate every previous organism's state. +is_deeply($trackList->{include}[0], "/a/jbrowse/tracks/tgonME49/tracks.conf", + 'the caller\'s skeleton is left untouched'); +is(scalar(@{$trackList->{tracks}}), 1, 'and so is its tracks array'); + +my @warnings = $G->warnings(); +$G->buildTrackList({include => ['/a/service/jbrowse/somethingNew/x']}, 'x', $BASE); +ok(scalar(grep { /somethingNew/ } $G->warnings()), + 'an unrecognised include is reported, not silently dropped'); + +# --------------------------------------------------------------------------- +# refSeqs.json derived from the .fai +# --------------------------------------------------------------------------- + +my $fai = "TGME49_chrIa\t1876705\t14\t60\t61\nTGME49_chrIb\t2199384\t1907792\t60\t60\n"; +is_deeply($G->refSeqsFromFai($fai), + [{name => 'TGME49_chrIa', start => 0, end => 1876705, length => 1876705}, + {name => 'TGME49_chrIb', start => 0, end => 2199384, length => 2199384}], + 'refSeqs derived from the fasta index, in index order'); + +# Numbers, not strings: JBrowse compares these arithmetically. +like(JSON->new->canonical->encode($G->refSeqsFromFai($fai)), qr/"end":1876705/, + 'lengths encode as JSON numbers'); + +is_deeply($G->refSeqsFromFai(""), [], 'an empty index yields an empty list'); + +eval { $G->refSeqsFromFai("TGME49_chrIa\n") }; +like($@, qr/malformed/i, 'a truncated index line is refused rather than read as length 0'); + +# --------------------------------------------------------------------------- +# useAsRefSeqStore stripping +# --------------------------------------------------------------------------- + +# Filter on the FIELD, not the label: the label is cosmetic and one organism +# renaming it would silently leave the store-URL reference track in place, +# fighting Apollo's own local copy. +my $withRefSeq = {tracks => [ + {label => 'refseqs', useAsRefSeqStore => JSON::true, storeClass => 'X'}, + {label => 'refseqs', storeClass => 'Y'}, + {label => 'notRefSeq', useAsRefSeqStore => JSON::true, storeClass => 'Z'}, + {label => 'genes', storeClass => 'W'}, +]}; +my ($stripped, $removed) = $G->stripRefSeqStoreTracks($withRefSeq); +is($removed, 2, 'both useAsRefSeqStore tracks are stripped regardless of label'); +is_deeply([map { $_->{label} } @{$stripped->{tracks}}], ['refseqs', 'genes'], + 'and nothing else is'); +is(scalar(@{$withRefSeq->{tracks}}), 4, 'the input document is not mutated'); + +# --------------------------------------------------------------------------- +# every track must be a JSON object +# --------------------------------------------------------------------------- + +# addChipChipTracks appended bare integers (56 of 158 entries for tgonME49). +# Fixed upstream; this is the last gate before a curator sees the output. +eval { $G->assertTracksAreObjects({tracks => [{a => 1}, 42, {b => 2}]}, 'chipseq.json') }; +like($@, qr/chipseq\.json/, 'a bare scalar in a tracks array is a hard failure'); +like($@, qr/\b1\b/, 'and the offending index is named'); +ok($G->assertTracksAreObjects({tracks => [{a => 1}]}, 'ok.json'), + 'a well formed tracks array passes'); +is($G->countTracks({tracks => [{a => 1}, {b => 2}]}), 2, 'track count'); +is($G->countTracks({tracks => []}), 0, + 'zero tracks is a count, not an error -- cneoJEC21 legitimately has none'); + +# --------------------------------------------------------------------------- +# writeFile: absolutization plus its post-condition +# --------------------------------------------------------------------------- + +my $dir = tempdir(CLEANUP => 1); + +$G->writeFile("$dir/t.json", '{"u":"/a/service/x","f":"seq/tgonME49.fa.fai"}', $BASE); +open(my $fh, '<', "$dir/t.json") or die $!; +my $written = do { local $/; <$fh> }; +close $fh; +is($written, '{"u":"https://veupathdb.org/a/service/x","f":"seq/tgonME49.fa.fai"}', + 'site-relative URLs absolutized; a bare relative seq/ path is left alone'); + +# The encoder must return OCTETS. Real track configs carry non-ASCII (tgonME49 +# does), and encoding to characters writes the right bytes but emits a "Wide +# character" warning on the way -- which this module elsewhere treats as proof +# that a config producer's output cannot be trusted. +my $wide = $G->encodeJson({k => "\x{2013}"}); +ok(!utf8::is_utf8($wide), 'encodeJson returns octets, not characters'); + +my @caught; +{ + local $SIG{__WARN__} = sub { push @caught, @_ }; + $G->writeFile("$dir/wide.json", $wide, $BASE); +} +is_deeply(\@caught, [], 'writing non-ASCII content produces no warnings'); +is(-s "$dir/wide.json", length($wide), 'and the bytes on disk are the bytes encoded'); + +# --------------------------------------------------------------------------- +# per-organism failure isolation +# --------------------------------------------------------------------------- + +my $results = $G->generateAll( + [{abbrev => 'good1'}, {abbrev => 'explodes'}, {abbrev => 'good2'}], + sub { + my ($organism) = @_; + die "simulated failure\n" if $organism->{abbrev} eq 'explodes'; + return {tracks => {rnaseq => 3}}; + }, +); +is_deeply([sort @{$results->{failed}}], ['explodes'], + 'a failing organism is recorded and the run continues'); +is_deeply([sort @{$results->{succeeded}}], ['good1', 'good2'], + 'the organisms either side of it still complete'); +like($results->{errors}{explodes}, qr/simulated failure/, 'the error text is kept'); + +done_testing(); From 5e414352e0f33697ee3d2f700e2c0831d784aafd Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 22:47:52 -0400 Subject: [PATCH 31/42] Apollo spec: record provenance and settle apollo_gene_tracks.conf JBrowse/bin/dumpConfigurationsForApollo.pl is the 2021 ancestor of the script being replaced; Paul's copy forked from it into a home directory and drifted there while the in-repo original rotted. That is precisely the failure section 8 corrects by putting the tool back in the repo. Its history also settles apollo_gene_tracks.conf: the in-repo version includes it unconditionally, and the 2021 "only dump annotated genomes" commit is the TODO being satisfied. The gate now lives in Portal::qualifies. The file is static, checked in, and required -- it defines the Draggable Annotation track curators drag genes into. Also marks the organismSpecific track counts in the section 10 table as superseded by the addChipChipTracks fix. Co-Authored-By: Claude Opus 5 --- ...026-08-21-apollo-release-package-design.md | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 3410bcf0db..84230e7940 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -380,14 +380,14 @@ Every script was run directly; nothing was built (`bld`/`wb` deliberately not ru | jbrowseRnaAndChipSeqTracks RNASeq | tgonME49 | 0 | 935200 | 0 | yes | 474 | | jbrowseRnaAndChipSeqTracks ChIPSeq | tgonME49 | 0 | 30852 | 0 | yes | 21 | | jbrowseRNASeqJunctionTracks | tgonME49 | 0 | 18338 | 0 | yes | 4 | -| jbrowseOrganismSpecificTracks | tgonME49 | 0 | 146486 | 0 | yes | 158 | +| jbrowseOrganismSpecificTracks | tgonME49 | 0 | 146486 | 0 | yes | 158 → **101** after the fixes below | | jbrowseDNASeqTracks | tgonME49 | 0 | 310296 | 0 | yes | 192 | | jbrowseRefSeqs | tgonME49 | **255** | 0 | **211** | no | — | | jbrowseTracks geneAnnotationTracks | tgonME49 | 0 | 352 | 0 | yes | 0 (by design) | | jbrowseRnaAndChipSeqTracks RNASeq | pfal3D7 | 0 | 1899853 | 0 | yes | 880 | | jbrowseRnaAndChipSeqTracks ChIPSeq | pfal3D7 | 0 | 355151 | 0 | yes | 230 | | jbrowseRNASeqJunctionTracks | pfal3D7 | 0 | 18350 | 0 | yes | 4 | -| jbrowseOrganismSpecificTracks | pfal3D7 | 0 | 132196 | 0 | yes | 103 | +| jbrowseOrganismSpecificTracks | pfal3D7 | 0 | 132196 | 0 | yes | 103 → **74** after the fixes below | | jbrowseDNASeqTracks | pfal3D7 | 0 | 2513500 | 0 | yes | 1611 | | jbrowseRefSeqs | pfal3D7 | **255** | 0 | **211** | no | — | | jbrowseTracks geneAnnotationTracks | pfal3D7 | 0 | 351 | 0 | yes | 0 (by design) | @@ -450,6 +450,28 @@ without re-checking a wider organism sample. Not measured: `https://jbrestel.eupathdb.org/a/service/jbrowse/tracks/tgonME49/trackList.json` 307-redirects to EuPathDB autologin, so the live-site comparison was skipped. +### Provenance, and why this belongs in the repo + +`JBrowse/bin/dumpConfigurationsForApollo.pl` is the ancestor of the script this work +replaces. It was last touched **2021-03-22** ("only dump annotated genomes"). Paul's +`~/apollo_config/bld-71-createApolloReleasePackage_ALL.pl` is a fork of it that lived in a +home directory and evolved there through builds 68 and 71 — gaining twoBit generation, the +Apollo update commands, and the arrow→curl migration — while the in-repo copy rotted. + +That is the failure this design corrects, and the reason §8 puts the tool back in +`ApiCommonModel` rather than in the harness: a fork in a home directory drifts silently and +then dies with its author. The stale 2021 script should be removed once this tool ships, +but that is a separate change. + +Its history also settles `apollo_gene_tracks.conf`: the in-repo version pushes it into the +include list unconditionally, and the 2021 commit that restricted the run to annotated +genomes is the `# TODO: only include this for annotated genomes` being satisfied. The gate +now lives further upstream, in `Portal::qualifies`, so every organism reaching generation is +an annotated genome and re-checking here would be dead code. The file itself is static and +checked in at `Model/lib/jbrowse/apollo_gene_tracks.conf`; nothing generates it. Apollo needs +it, because it defines the `Draggable Annotation` track curators drag genes into — without it +the package renders read-only evidence and no annotation target. + ## 11. Deferred - **A real `apollo` application type.** When JBrowse2/Apollo3 arrives, `applicationType` From f8d27efd1252a05e16072d2f7f3cdb19efc43a74 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:01:47 -0400 Subject: [PATCH 32/42] fix(apollo): close the spec-review gaps in Generate Implements the [tracks.refseq] strip that spec section 6 states and section 10 warned not to assume away. Census on cedar (build 71, 2026-08-21): 0 of 835 auto_generated tracks.conf files carry the stanza, so it is a no-op today -- shipped anyway, with the census recorded in the code, because an unimplemented stated requirement reads to the next person exactly like a forgotten one. Line based rather than the previous regex, whose character class stopped at the first comment inside the stanza. Covers generateOrganism, including the public/internal abbrev split that affects only 37 of 831 organisms and so survives any spot check. Verified by mutation: feeding the track producers the public abbrev, and writing output under the internal one, each fail the suite. Tests assertToolsAvailable, and records in a comment that nothing here calls it by design -- the CLI does, once, before the loop. Replaces a vacuous assertion: the bad-entry fixture had its index equal to the bad-entry count, so qr/\b1\b/ matched the count and deleting the whole 'at index(es)' clause left it green. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Generate.pm | 78 +++++++- Model/t/generate.t | 215 ++++++++++++++++++++++- 2 files changed, 287 insertions(+), 6 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Generate.pm b/Model/lib/perl/ApolloRelease/Generate.pm index 918c705aa2..fcd4a52637 100644 --- a/Model/lib/perl/ApolloRelease/Generate.pm +++ b/Model/lib/perl/ApolloRelease/Generate.pm @@ -287,6 +287,62 @@ sub countTracks { return scalar @{$document->{tracks} || []}; } +# --------------------------------------------------------------------------- +# PURE: strip the [tracks.refseq] stanza from a tracks.conf +# --------------------------------------------------------------------------- + +# Same reason as stripRefSeqStoreTracks: Apollo reads its own local copy of the +# genome out of seq/, so a second declaration of the reference sequence store +# is at best redundant and at worst wins. +# +# CENSUS, 2026-08-21 (cedar, build 71, GUS_HOME eupathdb.jbrestel): +# 835 auto_generated//tracks.conf files +# 0 contain "[tracks.refseq]" +# 0 contain the substring "refseq" at all, case-insensitively +# So this is a NO-OP today -- over the entire roster, not the four-organism +# sample the spec baseline used and warned not to generalise from. It is +# implemented anyway, deliberately: +# - spec section 6 states the requirement, and an unimplemented stated +# requirement is indistinguishable to the next reader from a forgotten one. +# That is exactly how the script this replaces accumulated its dead +# branches; +# - the stanza is emitted by the model, not by this tool, so it can come back +# in any build without a line here changing; +# - it costs one pass over a 35KB file. +# The count is returned so the caller can report a re-appearance rather than +# silently absorb it. +# +# Line based rather than Paul's `s/\[tracks.refseq\][^#\[]*//`, whose character +# class stops at the first "#" -- and tracks.conf is full of commented-out keys, +# so that regex leaves the tail of the stanza behind whenever one contains a +# comment. +sub stripRefSeqStanza { + my ($class, $text) = @_; + + return ($text, 0) unless defined $text; + + my @kept; + my $removed = 0; + my $inStanza = 0; + + # -1 limit so a trailing newline survives the rejoin. + foreach my $line (split /\n/, $text, -1) { + if ($line =~ /^\s*\[tracks\.refseq\]\s*$/) { + $inStanza = 1; + $removed++; + next; + } + + # A stanza runs to the next section header, or to end of file. Blank + # lines and comments inside it belong to it. + $inStanza = 0 if $inStanza && $line =~ /^\s*\[/; + + push @kept, $line unless $inStanza; + } + + return (join("\n", @kept), $removed); +} + # --------------------------------------------------------------------------- # IO: write one file, absolutized, with the post-condition enforced # --------------------------------------------------------------------------- @@ -370,6 +426,12 @@ sub _defaultTwoBit { } # Checked once, at startup, not 400 organisms into a run. +# +# DELIBERATELY NOT CALLED FROM THIS MODULE. generateOrganism is invoked once +# per organism, so a check here would either run 831 times or be the 400th +# organism's problem; and a caller substituting the `twoBit` seam has no use +# for faToTwoBit at all. The CLI (task 12) calls this once before the loop. +# It is tested here so that "uncalled" cannot quietly become "unverified". sub assertToolsAvailable { my ($class) = @_; @@ -487,9 +549,19 @@ sub generateOrganism { } # --- tracks.conf, keyed off the INTERNAL abbrev ------------------------- - $class->writeFile("$organismDir/tracks.conf", - $readFile->("$gusHome/lib/jbrowse/auto_generated/$internal/tracks.conf"), - $base); + my ($tracksConf, $stanzas) = $class->stripRefSeqStanza( + $readFile->("$gusHome/lib/jbrowse/auto_generated/$internal/tracks.conf")); + + # Zero for all 835 organisms as of 2026-08-21 (see stripRefSeqStanza). Say + # so if that changes, because it means the model started emitting a reference + # store again and someone should decide whether Apollo still wants it gone. + push @WARNINGS, "$abbrev: tracks.conf carried $stanzas [tracks.refseq] stanza(s), " + . "which were stripped; this has been a no-op for the whole roster since 2026-08-21" + if $stanzas; + + $summary{refseq_stanzas_stripped} = $stanzas; + + $class->writeFile("$organismDir/tracks.conf", $tracksConf, $base); push @generated, 'tracks.conf'; # --- static config ------------------------------------------------------ diff --git a/Model/t/generate.t b/Model/t/generate.t index d9cc3394db..ad04afcf1e 100644 --- a/Model/t/generate.t +++ b/Model/t/generate.t @@ -84,7 +84,6 @@ is_deeply($trackList->{include}[0], "/a/jbrowse/tracks/tgonME49/tracks.conf", 'the caller\'s skeleton is left untouched'); is(scalar(@{$trackList->{tracks}}), 1, 'and so is its tracks array'); -my @warnings = $G->warnings(); $G->buildTrackList({include => ['/a/service/jbrowse/somethingNew/x']}, 'x', $BASE); ok(scalar(grep { /somethingNew/ } $G->warnings()), 'an unrecognised include is reported, not silently dropped'); @@ -133,15 +132,59 @@ is(scalar(@{$withRefSeq->{tracks}}), 4, 'the input document is not mutated'); # addChipChipTracks appended bare integers (56 of 158 entries for tgonME49). # Fixed upstream; this is the last gate before a curator sees the output. -eval { $G->assertTracksAreObjects({tracks => [{a => 1}, 42, {b => 2}]}, 'chipseq.json') }; +# The bad entries sit at indexes 2 and 4 of 5, so neither index can be +# satisfied by the bad-count (2) or the total (5). The earlier fixture -- one +# bad entry, at index 1 -- produced "1 of 3 entries ... at index(es) 1", which +# matched qr/\b1\b/ on the count alone: deleting the whole "at index(es)" +# clause from the message left this test green. +eval { $G->assertTracksAreObjects( + {tracks => [{a => 1}, {b => 2}, 42, {c => 3}, 'nope']}, 'chipseq.json') }; like($@, qr/chipseq\.json/, 'a bare scalar in a tracks array is a hard failure'); -like($@, qr/\b1\b/, 'and the offending index is named'); +like($@, qr/index\(es\)\s+2,\s*4\b/, 'and every offending index is named'); +like($@, qr/\b2 of 5\b/, 'reported against the total, so the scale is visible'); ok($G->assertTracksAreObjects({tracks => [{a => 1}]}, 'ok.json'), 'a well formed tracks array passes'); is($G->countTracks({tracks => [{a => 1}, {b => 2}]}), 2, 'track count'); is($G->countTracks({tracks => []}), 0, 'zero tracks is a count, not an error -- cneoJEC21 legitimately has none'); +# --------------------------------------------------------------------------- +# [tracks.refseq] stanza stripping +# --------------------------------------------------------------------------- + +# A no-op over the whole roster today (0 of 835 tracks.conf files on +# 2026-08-21), so this fixture IS the only example of the shape. The comment +# in stripRefSeqStanza records why it ships anyway. +my $conf = join "\n", + '[general]', + 'dataset_id=tgonME49', + '', + '[tracks.refseq]', + 'storeClass=JBrowse/Store/SeqFeature/IndexedFasta', + '#urlTemplate=commented out, and Paul\'s regex stopped here', + 'faiUrlTemplate=/a/service/jbrowse/store?data=x.fai', + '', + '[tracks.gcContent]', + 'key=GC Content', + ''; + +my ($cleaned, $stanzas) = $G->stripRefSeqStanza($conf); +is($stanzas, 1, 'the refseq stanza is counted'); +unlike($cleaned, qr/refseq/i, 'and every line of it is gone'); + +# The specific failure of the previous implementation: its character class +# stopped at the first "#", leaving the rest of the stanza behind. +unlike($cleaned, qr/faiUrlTemplate/, + 'including the keys after a comment line inside the stanza'); +like($cleaned, qr/\[tracks\.gcContent\]\nkey=GC Content/, + 'the following stanza survives intact'); +like($cleaned, qr/^\[general\]\ndataset_id=tgonME49\n/, + 'and so does everything before it'); + +my ($untouched, $none) = $G->stripRefSeqStanza($conf =~ s/\[tracks\.refseq\]/[tracks.other]/r); +is($none, 0, 'a tracks.conf without the stanza reports zero'); +like($untouched, qr/faiUrlTemplate/, 'and is returned unchanged'); + # --------------------------------------------------------------------------- # writeFile: absolutization plus its post-condition # --------------------------------------------------------------------------- @@ -188,4 +231,170 @@ is_deeply([sort @{$results->{succeeded}}], ['good1', 'good2'], 'the organisms either side of it still complete'); like($results->{errors}{explodes}, qr/simulated failure/, 'the error text is kept'); +# --------------------------------------------------------------------------- +# assertToolsAvailable +# --------------------------------------------------------------------------- + +# Not called by generateOrganism -- the CLI calls it once before the loop -- so +# without this it would be shipped unverified. PATH is manipulated rather than +# trusting the host, so the result does not depend on whose cedar account runs +# the suite. +{ + my $binDir = tempdir(CLEANUP => 1); + local $ENV{PATH} = $binDir; + + my $err = do { local $@; eval { $G->assertToolsAvailable() }; $@ }; + like($err, qr/faToTwoBit is not on PATH/, 'a missing faToTwoBit fails immediately'); + like($err, qr{~/bin/faToTwoBit}, + 'and the message carries the install location, not just the complaint'); + like($err, qr/libssl\.so\.10/, + 'and warns off the 2016 yew binary, which is the obvious wrong fix'); + + open(my $fake, '>', "$binDir/faToTwoBit") or die $!; + close $fake; + chmod 0755, "$binDir/faToTwoBit"; + is($G->assertToolsAvailable(), "$binDir/faToTwoBit", + 'and it returns the resolved path when present'); +} + +# --------------------------------------------------------------------------- +# generateOrganism: the public/internal abbrev split, end to end +# --------------------------------------------------------------------------- + +# The one function that wires the two abbrev namespaces together, and the +# mistake it guards against is invisible on 794 of 831 organisms. So the +# fixture is a renamed one, where public and internal genuinely differ. +{ + my $out = tempdir(CLEANUP => 1); + + my @scriptCalls; + my @reads; + my @copies; + my @twoBits; + + my $fai = "chr1\t100\t10\t60\t61\nchr2\t250\t200\t60\t61\n"; + + my $summary = $G->generateOrganism( + {abbrev => 'cdenJEC21', internal_abbrev => 'cneoJEC21', + name_for_filenames => 'CdeneoformansJEC21'}, + { + outDir => $out, + base => $BASE, + project => 'UniDB', + build => 71, + wsDir => '/ws', + gusHome => '/gus', + + runScript => sub { + my ($gusHome, $script, @args) = @_; + push @scriptCalls, {script => $script, args => \@args}; + + return encode_json({ + names => {type => 'REST', url => '/a/service/jbrowse/names/cdenJEC21'}, + tracks => [], + refSeqs => '/a/service/jbrowse/store?data=x', + include => ['/a/jbrowse/tracks/cdenJEC21/tracks.conf'], + }) if $script eq 'jbrowseTracks'; + + # organismSpecific carries the store-URL reference track; the others + # carry one ordinary track each. + return encode_json({tracks => [ + {label => 'refseqs', useAsRefSeqStore => JSON::true}, + {label => 'real', urlTemplate => '/a/service/jbrowse/store?data=y'}, + ]}) if $script eq 'jbrowseOrganismSpecificTracks'; + + return encode_json({tracks => [{label => 'real'}]}); + }, + + readFile => sub { + my ($path) = @_; + push @reads, $path; + return $fai if $path =~ /\.fa\.fai$/; + return "[tracks.x]\nkey=from $path\n"; + }, + + copyFile => sub { push @copies, [@_]; return 1 }, + twoBit => sub { push @twoBits, [@_]; return 1 }, + }, + ); + + # --- the five track producers take the INTERNAL abbrev ------------------ + my @producers = grep { $_->{script} ne 'jbrowseTracks' } @scriptCalls; + is(scalar(@producers), 5, 'all five track producers ran'); + is_deeply([map { $_->{args}[0] } @producers], + [('cneoJEC21') x 5], + 'each track producer received the INTERNAL abbrev'); + is_deeply([sort map { $_->{script} } @producers], + ['jbrowseDNASeqTracks', 'jbrowseOrganismSpecificTracks', + 'jbrowseRNASeqJunctionTracks', 'jbrowseRnaAndChipSeqTracks', + 'jbrowseRnaAndChipSeqTracks'], + 'and they are the five the spec names'); + + # --- jbrowseTracks takes the PUBLIC abbrev ------------------------------ + my ($skeleton) = grep { $_->{script} eq 'jbrowseTracks' } @scriptCalls; + is($skeleton->{args}[0], 'cdenJEC21', + 'jbrowseTracks received the PUBLIC abbrev -- its SQL matches public_abbrev'); + is_deeply($skeleton->{args}, ['cdenJEC21', 'UniDB', 0, 'geneAnnotationTracks'], + 'with the geneAnnotationTracks skeleton arguments'); + + # --- tracks.conf is keyed off the INTERNAL abbrev ----------------------- + ok(scalar(grep { $_ eq '/gus/lib/jbrowse/auto_generated/cneoJEC21/tracks.conf' } @reads), + 'tracks.conf read from auto_generated//'); + is(scalar(grep { m{auto_generated/cdenJEC21} } @reads), 0, + 'and never from auto_generated//, which does not exist'); + + # --- every OUTPUT path uses the PUBLIC abbrev --------------------------- + is_deeply(\@copies, + [['/ws/UniDB/build-71/CdeneoformansJEC21/genomeAndProteome/fasta/genome.fasta', + "$out/data/cdenJEC21/seq/cdenJEC21.fa"], + ['/ws/UniDB/build-71/CdeneoformansJEC21/genomeAndProteome/fasta/genome.fasta.fai', + "$out/data/cdenJEC21/seq/cdenJEC21.fa.fai"]], + 'the genome is sourced by name_for_filenames and lands under the PUBLIC abbrev'); + is_deeply(\@twoBits, + [["$out/data/cdenJEC21/seq/cdenJEC21.fa", "$out/twoBit/cdenJEC21.2bit"]], + 'and so does the .2bit'); + ok(-f "$out/data/cdenJEC21/trackList.json", 'the organism dir is the PUBLIC abbrev'); + ok(!-e "$out/data/cneoJEC21", 'nothing is written under the internal abbrev'); + + # --- and the package it produced ---------------------------------------- + my $written = decode_json(do { + open(my $t, '<', "$out/data/cdenJEC21/trackList.json") or die $!; + local $/; <$t>; + }); + is($written->{refSeqs}, 'seq/cdenJEC21.fa.fai', 'trackList refSeqs uses the public abbrev'); + is($written->{tracks}[0]{urlTemplate}, 'seq/cdenJEC21.fa', 'as does its reference track'); + is_deeply($written->{include}, + ['tracks.conf', 'rnaseq.json', 'chipseq.json', 'rnaseqJunctions.json', + 'organismSpecific.json', 'dnaseq.json', 'functions.conf', + 'apollo_gene_tracks.conf'], + 'every include names a file this run actually wrote'); + ok(-f "$out/data/cdenJEC21/$_", "$_ was written") for @{$written->{include}}; + + is($summary->{stripped}, 1, 'the store-URL reference track was stripped'); + is($summary->{sequences}, 2, 'sequence count comes from the copied index'); + is_deeply($summary->{counts}, + {'rnaseq.json' => 1, 'chipseq.json' => 1, 'rnaseqJunctions.json' => 1, + 'organismSpecific.json' => 1, 'dnaseq.json' => 1}, + 'track counts are per file, after stripping'); + is($summary->{refseq_stanzas_stripped}, 0, 'no [tracks.refseq] stanza in this fixture'); + + my $specific = decode_json(do { + open(my $t, '<', "$out/data/cdenJEC21/organismSpecific.json") or die $!; + local $/; <$t>; + }); + is(scalar(grep { $_->{useAsRefSeqStore} } @{$specific->{tracks}}), 0, + 'and no useAsRefSeqStore track survives into the file'); + is($specific->{tracks}[0]{urlTemplate}, 'https://veupathdb.org/a/service/jbrowse/store?data=y', + 'the surviving track was absolutized on the way out'); + + my $refSeqs = decode_json(do { + open(my $t, '<', "$out/data/cdenJEC21/seq/refSeqs.json") or die $!; + local $/; <$t>; + }); + is_deeply($refSeqs, + [{name => 'chr1', start => 0, end => 100, length => 100}, + {name => 'chr2', start => 0, end => 250, length => 250}], + 'refSeqs.json derived from the index that was copied, not re-queried'); +} + done_testing(); From 8deb571a9f400bb0b1f2f3a7ffcc04b2c0f1a832 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:14:39 -0400 Subject: [PATCH 33/42] feat(apollo): emit Apollo update, rename, prune and add commands Visibility is echoed from what Apollo currently holds, never defaulted; a rename targets the existing organism id so its annotations survive; prune unpublishes rather than deletes; passwords are emitted as a shell variable, never interpolated. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Commands.pm | 246 +++++++++++++++++++++++ Model/t/commands.t | 230 +++++++++++++++++++++ 2 files changed, 476 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Commands.pm create mode 100644 Model/t/commands.t diff --git a/Model/lib/perl/ApolloRelease/Commands.pm b/Model/lib/perl/ApolloRelease/Commands.pm new file mode 100644 index 0000000000..d68d2b334e --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Commands.pm @@ -0,0 +1,246 @@ +package ApiCommonModel::Model::ApolloRelease::Commands; + +use strict; +use warnings; + +# Emits the commands a human runs against Apollo once the release package has +# been synced. This module NEVER executes anything and never opens a socket: +# its whole output is text destined for two files. That is deliberate. The +# same inputs that produce a correct release produce, with one field wrong, an +# irreversible loss of curated annotations -- so the step between "computed" +# and "applied" is a file a human reads. +# +# Passwords are emitted as the shell variable $APOLLO_ADMIN_PASSWORD, never +# interpolated: these files land in a shared directory and get pasted into +# tickets. + +my $API = 'https://apollo-api.veupathdb.org'; +my $UI = 'https://apollo.apidb.org'; +my $DATA = '/data/apollo_data'; +my $ADMIN = 'admin@local.host'; + +# Every curl command is the same call -- updateOrganismInfo -- differing only +# in which id it names and what it sets. Keeping one emitter means a change to +# quoting, endpoint or credentials cannot apply to three of the four kinds. +# +# Field order is fixed rather than hash order so two runs over the same +# reconciliation produce byte-identical files; a diff between releases is then +# a diff of decisions, not of Perl's hash seed. +sub _updateOrganismInfo { + my ($class, $id, $abbrev, $extra) = @_; + + die "_updateOrganismInfo: no apollo id\n" unless defined $id && length $id; + die "_updateOrganismInfo: no abbrev\n" unless defined $abbrev && length $abbrev; + + my %fields = ( + username => $ADMIN, + password => '$APOLLO_ADMIN_PASSWORD', + id => "$id", + directory => "$DATA/$abbrev", + blatdb => "$DATA/twoBit/$abbrev.2bit", + %{$extra || {}}, + ); + + my @fixed = qw(username password id directory blatdb); + my %isFixed = map { $_ => 1 } @fixed; + my @order = (@fixed, grep { !$isFixed{$_} } sort keys %fields); + + # A single quote in a value would terminate the shell's quoting of --data and + # turn the rest of the JSON into shell words. commonName comes from the + # portal, so this is reachable in principle; refuse rather than emit a line + # whose meaning depends on the shell. + foreach my $key (@order) { + die "$key value contains a single quote, which the emitted shell command " + . "cannot quote safely: $fields{$key}\n" + if $fields{$key} =~ /'/; + die "$key value contains a double quote or backslash, which would break " + . "the emitted JSON: $fields{$key}\n" + if $fields{$key} =~ /["\\]/; + } + + my $json = join(',', map { qq{"$_":"$fields{$_}"} } @order); + + return qq{curl -X POST -H "Content-Type: application/json" } + . qq{--data '{$json}' $API/organism/updateOrganismInfo\n}; +} + +# Apollo names an organism " []". With no +# usable history row Portal leaves latest_annotation_version undef; emit the +# bare name rather than an empty "[]", which would become part of the +# organism's identity in Apollo and never match a later release. +sub apolloName { + my ($class, $organism) = @_; + + die "apolloName: no organism record\n" unless $organism; + my $name = $organism->{name}; + die "apolloName: organism has no name\n" unless defined $name && length $name; + + my $version = $organism->{latest_annotation_version}; + + return (defined $version && length $version) ? "$name [$version]" : $name; +} + +# An update must NOT change visibility. Publishing and unpublishing are +# curation decisions: 17 live organisms are deliberately hidden, three of them +# carrying annotations. The previous script hardcoded "publicMode":"true" +# here, which re-published every one of them. Echo back what Apollo currently +# holds -- and if we were not told what that is, die. A default is exactly how +# that bug returns, because a default looks correct in every test where the +# field happens to be present. +sub updateCommand { + my ($class, $entry) = @_; + + die "updateCommand: public_mode missing for $entry->{abbrev}; refusing to guess visibility\n" + unless defined $entry->{public_mode}; + + return $class->_updateOrganismInfo( + $entry->{apollo_id}, $entry->{abbrev}, + {publicMode => $entry->{public_mode} ? 'true' : 'false'}, + ); +} + +# A rename repoints the EXISTING organism: same id, new directory, new blatdb, +# new commonName. cneoJEC21 (id 2452162) holds 14 human-made annotations and +# has been reclassified as cdenJEC21; repointing that id preserves them. +# +# The wrong shape -- add the new abbrev, prune the old -- uses the same API and +# the same inputs, succeeds, and produces an empty organism plus 14 orphaned +# annotations. Nothing downstream can tell the two apart, which is why the id +# used here is the old organism's and the paths are the new abbrev's. +sub renameCommand { + my ($class, $entry) = @_; + + die "renameCommand: public_mode missing for $entry->{from_abbrev}; refusing to guess visibility\n" + unless defined $entry->{public_mode}; + + return $class->_updateOrganismInfo( + $entry->{apollo_id}, $entry->{to_abbrev}, + {publicMode => $entry->{public_mode} ? 'true' : 'false', + commonName => $class->apolloName($entry->{organism})}, + ); +} + +# Prune UNPUBLISHES. It is reversible by a single field flip, and Apollo's API +# has no delete in use. A prune is generated from the absence of an organism +# on the portal -- a weaker signal than a curator's decision to keep data -- so +# the action it triggers must be the recoverable one. +sub pruneCommand { + my ($class, $entry) = @_; + + return $class->_updateOrganismInfo( + $entry->{apollo_id}, $entry->{abbrev}, + {publicMode => 'false'}, + ); +} + +# An add creates an organism that does not exist in Apollo, then grants the +# remote_users group access to it. Unlike the curl commands this is NOT +# idempotent -- a second run adds a second organism with the same name -- which +# is why adds live in their own file and are never mixed into Apollo_curl. +sub addCommand { + my ($class, $entry) = @_; + + my $abbrev = $entry->{abbrev}; + die "addCommand: no abbrev\n" unless defined $abbrev && length $abbrev; + + my $name = $class->apolloName($entry->{organism}); + + die "addCommand: organism name contains a single quote and cannot be " + . "shell-quoted safely: $name\n" if $name =~ /'/; + + return + qq{groovy add_organism.groovy -name '$name' -url $UI } + . qq{-directory '$DATA/$abbrev' -blatdb '$DATA/twoBit/$abbrev.2bit' } + . qq{-username '$ADMIN' -password \$APOLLO_ADMIN_PASSWORD\n} + . qq{groovy alter_group_permissions.groovy -groupname remote_users } + . qq{-organism '$name' -permission WRITE -destinationurl $UI/ } + . qq{-adminusername '$ADMIN' -adminpassword \$APOLLO_ADMIN_PASSWORD\n}; +} + +# Writes the two files a human runs. Ordering is part of the contract: +# +# renames first -- they are the only commands whose target holds curated +# work, so if a run is interrupted they are the ones that +# have already landed; +# updates next -- pure repointing, no organism changes identity; +# prunes last -- an interrupted run then leaves everything published, +# which is the recoverable end state. +# +# ONLY APPROVED adds and prunes are emitted. An unapproved candidate is a +# proposal for a human, reported by Report.pm; turning it into a runnable line +# would make the roster overlay advisory rather than the gate it is. +sub writeCommandFiles { + my ($class, $result, $dir, %opts) = @_; + + die "writeCommandFiles: no output directory\n" unless defined $dir && length $dir; + + my @renames = @{$result->{rename} || []}; + my @updates = @{$result->{update} || []}; + my @prunes = grep { $_->{approved} } @{$result->{prune_candidate} || []}; + my @adds = grep { $_->{approved} } @{$result->{add_candidate} || []}; + + my $curlPath = "$dir/Apollo_curl"; + my $groovyPath = "$dir/Apollo_groovy"; + + open(my $curl, '>', $curlPath) or die "Cannot write $curlPath: $!"; + print $curl $class->_header( + 'Apollo_curl', + \%opts, + sprintf('%d rename(s), %d update(s), %d approved prune(s)', + scalar @renames, scalar @updates, scalar @prunes), + 'Order matters: renames, then updates, then prunes. Run the file top to', + 'bottom. Re-running it is safe -- every line sets the same fields to the', + 'same values.', + ); + print $curl $class->renameCommand($_) for @renames; + print $curl $class->updateCommand($_) for @updates; + foreach my $prune (@prunes) { + # An approved prune of an annotated organism is still emitted: approval is + # the human gate, and unpublishing is reversible. But it is never silent + # -- whoever runs the file is told, on the line above, what is about to + # become invisible. + my $count = $prune->{annotation_count} || 0; + print $curl "# WARNING: $prune->{abbrev} has $count human annotation(s); " + . "unpublishing hides them.\n" if $count; + print $curl $class->pruneCommand($prune); + } + close $curl or die "Cannot close $curlPath: $!"; + + open(my $groovy, '>', $groovyPath) or die "Cannot write $groovyPath: $!"; + print $groovy $class->_header( + 'Apollo_groovy', + \%opts, + sprintf('%d approved add(s)', scalar @adds), + 'NOT idempotent: running this file twice creates duplicate organisms.', + ); + print $groovy $class->addCommand($_) for @adds; + close $groovy or die "Cannot close $groovyPath: $!"; + + return {curl => $curlPath, groovy => $groovyPath}; +} + +# The header is comment lines, so the file stays runnable as a shell script. +# It exists because these files outlive the run that made them: they are copied +# into tickets and read weeks later, at which point "which build is this, and +# does it still apply?" has no other answer. The date is caller-supplied where +# possible so a regenerated package is byte-comparable with its predecessor. +sub _header { + my ($class, $fileName, $opts, @lines) = @_; + + my $date = $opts->{date}; + unless (defined $date) { + my @t = gmtime(time); + $date = sprintf('%04d-%02d-%02d', $t[5] + 1900, $t[4] + 1, $t[3]); + } + + my $build = defined $opts->{build} ? $opts->{build} : 'unspecified'; + + return join('', map { "# $_\n" } + "$fileName -- generated by createApolloReleasePackage", + "build $build, generated $date", + @lines, + 'Passwords are not stored here: export APOLLO_ADMIN_PASSWORD before running.', + ) . "#\n"; +} + +1; diff --git a/Model/t/commands.t b/Model/t/commands.t new file mode 100644 index 0000000000..870a14484e --- /dev/null +++ b/Model/t/commands.t @@ -0,0 +1,230 @@ +use strict; +use warnings; +use Test::More; +use File::Temp qw(tempdir); +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Commands; + +my $C = 'ApiCommonModel::Model::ApolloRelease::Commands'; + +sub slurp { + my ($path) = @_; + open(my $fh, '<', $path) or die "Cannot read $path: $!"; + local $/; + my $content = <$fh>; + close $fh; + return $content; +} + +# ---------------------------------------------------------------- update + +my $update = $C->updateCommand({ + abbrev => 'tgonME49', apollo_id => 1484940, public_mode => 1, +}); + +like($update, qr/"id":"1484940"/, 'update targets the numeric apollo id'); +like($update, qr{"directory":"/data/apollo_data/tgonME49"}, + 'update points at the organism directory'); +like($update, qr{"blatdb":"/data/apollo_data/twoBit/tgonME49\.2bit"}, + 'update points at the twoBit blatdb'); +like($update, qr{\Qhttps://apollo-api.veupathdb.org/organism/updateOrganismInfo\E}, + 'update posts to updateOrganismInfo'); +like($update, qr/"publicMode":"true"/, 'a public organism stays public through an update'); + +# --------------------------------------------------- visibility is echoed + +# 17 live organisms are curator-hidden, 3 of them carrying annotations. The +# previous script hardcoded "publicMode":"true", which re-published all 17. +my $hidden = $C->updateCommand({abbrev => 'treeQM6a', apollo_id => 9999, public_mode => 0}); +like($hidden, qr/"publicMode":"false"/, + 'a hidden organism stays hidden through an update'); +unlike($hidden, qr/"publicMode":"true"/, + 'and the hidden update contains no publicMode:true anywhere'); + +eval { $C->updateCommand({abbrev => 'x', apollo_id => 1}) }; +like($@, qr/refusing to guess visibility/, + 'a missing public_mode is a hard error, not a default'); + +eval { $C->renameCommand({from_abbrev => 'a', to_abbrev => 'b', apollo_id => 1, + organism => {name => 'A b'}}) }; +like($@, qr/refusing to guess visibility/, + 'a rename with no public_mode is a hard error too'); + +# ---------------------------------------------------------------- rename + +# cneoJEC21 (id 2452162) holds 14 human-made annotations and is now cdenJEC21. +# Repointing the EXISTING id preserves them; add-new + prune-old orphans them. +my $rename = $C->renameCommand({ + from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', + apollo_id => 2452162, public_mode => 1, annotation_count => 14, + organism => {name => 'Cryptococcus deneoformans JEC21', + latest_annotation_version => 'Jun 16, 2016'}, +}); + +like($rename, qr/"id":"2452162"/, + 'a rename targets the EXISTING apollo id, so annotations survive'); +unlike($rename, qr/cneoJEC21/, + 'and the OLD abbrev appears nowhere in the paths'); +like($rename, qr{"directory":"/data/apollo_data/cdenJEC21"}, + 'a rename repoints directory at the new abbrev'); +like($rename, qr{"blatdb":"/data/apollo_data/twoBit/cdenJEC21\.2bit"}, + 'a rename repoints blatdb at the new abbrev'); +like($rename, qr/"commonName":"Cryptococcus deneoformans JEC21 \[Jun 16, 2016\]"/, + 'a rename sets commonName to " []"'); +like($rename, qr/"publicMode":"true"/, 'a rename echoes back the current visibility'); + +my $hiddenRename = $C->renameCommand({ + from_abbrev => 'a', to_abbrev => 'b', apollo_id => 7, public_mode => 0, + organism => {name => 'Some organism', latest_annotation_version => '2020-01-01'}, +}); +like($hiddenRename, qr/"publicMode":"false"/, 'a hidden organism stays hidden through a rename'); + +# ----------------------------------------------------------------- prune + +my $prune = $C->pruneCommand({abbrev => 'cglaCBS138', apollo_id => 5146948}); +like($prune, qr/"publicMode":"false"/, 'a prune unpublishes'); +unlike($prune, qr/"publicMode":"true"/, 'a prune never publishes'); +unlike($prune, qr/\b(delete|remove|deleteOrganism)\b/i, + 'a prune carries no delete verb -- unpublishing is reversible'); +like($prune, qr{/organism/updateOrganismInfo}, + 'a prune goes through updateOrganismInfo like everything else'); + +# ------------------------------------------------------------------- add + +my $add = $C->addCommand({ + abbrev => 'pberANKA', + approved => 1, + organism => {name => 'Plasmodium berghei ANKA', + latest_annotation_version => 'Mar 2021'}, +}); + +like($add, qr/^groovy add_organism\.groovy /m, 'an add emits add_organism.groovy'); +like($add, qr/^groovy alter_group_permissions\.groovy /m, + 'and alter_group_permissions.groovy alongside it'); +like($add, qr/-name 'Plasmodium berghei ANKA \[Mar 2021\]'/, + 'the new organism is named " []"'); +like($add, qr{-directory '/data/apollo_data/pberANKA'}, 'the add names the organism directory'); +like($add, qr{-blatdb '/data/apollo_data/twoBit/pberANKA\.2bit'}, 'and its blatdb'); + +# ------------------------------------------- an organism with no annotation version + +# _latestAnnotationVersion returns undef when an organism has no usable history +# row. Apollo's name is then the bare portal name: an empty "[]" would become +# part of the organism's identity in Apollo and would never match again. +my $noVersion = $C->addCommand({ + abbrev => 'xxxNOVER', approved => 1, organism => {name => 'Genus species NOVER'}, +}); +like($noVersion, qr/-name 'Genus species NOVER'/, + 'no annotation version -> the bare portal name'); +unlike($noVersion, qr/\[|\]/, + 'and no empty brackets, which would be baked into the Apollo identity'); + +my $noVersionRename = $C->renameCommand({ + from_abbrev => 'old', to_abbrev => 'new', apollo_id => 42, public_mode => 1, + organism => {name => 'Genus species NOVER'}, +}); +like($noVersionRename, qr/"commonName":"Genus species NOVER"/, + 'a rename with no annotation version uses the bare name'); +unlike($noVersionRename, qr/\[|\]/, 'and emits no empty brackets either'); + +# ------------------------------------------------------------- no passwords + +# These files land in a shared directory and get pasted into tickets. Every +# password-bearing field must hold the literal shell variable, unexpanded. +sub passwordValues { + my ($text) = @_; + my @values; + while ($text =~ /(?:"password"\s*:\s*"([^"]*)"|-adminpassword\s+(\S+)|-password\s+(\S+))/g) { + push @values, defined $1 ? $1 : defined $2 ? $2 : $3; + } + return @values; +} + +for my $pair (['update', $update], ['rename', $rename], ['prune', $prune], ['add', $add]) { + my ($what, $text) = @$pair; + my @values = passwordValues($text); + ok(scalar @values, "$what carries at least one password field"); + is_deeply([grep { $_ ne '$APOLLO_ADMIN_PASSWORD' } @values], [], + "$what interpolates no literal password -- only \$APOLLO_ADMIN_PASSWORD"); +} + +like($update, qr/"username":"admin\@local\.host"/, 'the admin user is admin@local.host'); + +# ---------------------------------------------------- writing the two files + +my $organism = sub { + my ($name, $version) = @_; + return {name => $name, latest_annotation_version => $version}; +}; + +my $result = { + update => [ + {abbrev => 'tgonME49', apollo_id => 1484940, public_mode => 1, + organism => $organism->('Toxoplasma gondii ME49', '2021-05-01')}, + {abbrev => 'treeQM6a', apollo_id => 9999, public_mode => 0, + organism => $organism->('Trichoderma reesei QM6a', '2019-01-01')}, + ], + rename => [ + {from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', apollo_id => 2452162, + public_mode => 1, annotation_count => 14, + organism => $organism->('Cryptococcus deneoformans JEC21', 'Jun 16, 2016')}, + ], + prune_candidate => [ + {abbrev => 'cglaCBS138', apollo_id => 5146948, annotation_count => 0, + approved => 1, reason => 'no longer a reference strain'}, + {abbrev => 'unapprovedP', apollo_id => 111, annotation_count => 0, approved => 0}, + ], + add_candidate => [ + {abbrev => 'pberANKA', approved => 1, reason => 'new reference', + organism => $organism->('Plasmodium berghei ANKA', 'Mar 2021')}, + {abbrev => 'unapprovedA', approved => 0, + organism => $organism->('Genus unapproved', 'Jan 2020')}, + ], +}; + +my $dir = tempdir(CLEANUP => 1); +$C->writeCommandFiles($result, $dir, build => '68', date => '2026-08-21'); + +my $curl = slurp("$dir/Apollo_curl"); +my $groovy = slurp("$dir/Apollo_groovy"); + +unlike($curl, qr/unapprovedP/, 'an UNAPPROVED prune produces no command'); +like($curl, qr/"id":"5146948"/, 'an approved prune does'); +unlike($groovy, qr/unapprovedA/, 'an UNAPPROVED add produces no command'); +like($groovy, qr/pberANKA/, 'an approved add does'); + +my @commands = grep { !/^\s*#/ && /\S/ } split(/\n/, $curl); +is(scalar @commands, 4, + 'Apollo_curl holds exactly updates + renames + APPROVED prunes'); + +# A partial run should leave everything published: the renames (the entries +# carrying annotations) go first, the unpublishes last. +like($commands[0], qr/2452162/, 'renames are emitted first'); +like($commands[3], qr/5146948/, 'prunes are emitted last'); + +like($curl, qr/^#/m, 'Apollo_curl carries a header comment'); +like($curl, qr/2026-08-21/, 'the header records the date it was generated'); +like($curl, qr/\b68\b/, 'and the build it belongs to'); +like($groovy, qr/^#/m, 'Apollo_groovy carries a header comment too'); + +# An approved prune of an annotated organism is still emitted -- approval is the +# human gate and unpublishing is reversible -- but never silently. +my $annotated = { + update => [], rename => [], add_candidate => [], + prune_candidate => [ + {abbrev => 'cneoOLD', apollo_id => 777, annotation_count => 14, approved => 1, + reason => 'superseded'}, + ], +}; +my $dir2 = tempdir(CLEANUP => 1); +$C->writeCommandFiles($annotated, $dir2); +my $curl2 = slurp("$dir2/Apollo_curl"); +like($curl2, qr/"id":"777"/, 'an approved prune of an annotated organism is emitted'); +like($curl2, qr/#[^\n]*14[^\n]*annotation/i, + 'but is preceded by a comment naming its annotation count'); + +for my $text ($curl, $groovy, $curl2) { + unlike($text, qr/APOLLO_ADMIN_PASSWORD=/, 'no generated file assigns a password value'); +} + +done_testing(); From 868843666d1fa4cb0ee90bb722d10c9d85d6eca5 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:15:58 -0400 Subject: [PATCH 34/42] Apollo: command-file verification counts non-comment lines; flag groovy URLs The command files now carry a header and a WARNING line above each annotated prune, so the plan's line-count check must ignore comments. Also records a pre-flight check: add_organism.groovy is invoked with -url and alter_group_permissions.groovy with -destinationurl plus a trailing slash. Both came from the legacy script, which is inconsistent the same way, and neither groovy script is in these repos. An add is the one non-idempotent command, so this is worth confirming before the first real one. Co-Authored-By: Claude Opus 5 --- .../plans/2026-08-21-apollo-release-package.md | 2 +- .../specs/2026-08-21-apollo-release-package-design.md | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-08-21-apollo-release-package.md b/docs/superpowers/plans/2026-08-21-apollo-release-package.md index be06a2d530..2e6435df2a 100644 --- a/docs/superpowers/plans/2026-08-21-apollo-release-package.md +++ b/docs/superpowers/plans/2026-08-21-apollo-release-package.md @@ -2442,7 +2442,7 @@ Expect hours. Exit status is non-zero if any organism failed; the summary names - every organism dir has the nine files and a `seq/` with three - `twoBit/` count equals the roster size - `report.txt` is present in the release dir -- `Apollo_curl` line count equals updates + renames + approved prunes +- `Apollo_curl` line count equals updates + renames + approved prunes — **counting only non-`#` lines**; the file carries a header and a `# WARNING` line above each annotated prune - `Apollo_groovy` contains only approved adds - [ ] **Step 4: Sandbox the rename before prod** diff --git a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md index 84230e7940..d0c9851f11 100644 --- a/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md +++ b/docs/superpowers/specs/2026-08-21-apollo-release-package-design.md @@ -485,6 +485,14 @@ the package renders read-only evidence and no annotation target. so after a rename `commonName` will say *deneoformans* while those fields may not. Determine whether the Apollo GUI keys off them; if so, this becomes part of the rename command. +- **Verify the groovy invocation before the first real add.** `add_organism.groovy` is + called with `-url https://apollo.apidb.org` while `alter_group_permissions.groovy` is + called with `-destinationurl https://apollo.apidb.org/` — different flag names and a + trailing-slash difference. Both were carried forward from the legacy script, which shows + the same inconsistency, and neither groovy script lives in these repos so it cannot be + checked here. Confirm against the Apollo-side scripts before running an add for real; an + add is the one command that is **not** idempotent. + - **`--environment qa`.** Retained from the old script and still generated, but the qa and prod rosters have never differed by anything but staleness. Revisit whether qa is worth producing at all. From 49621e7118b9505edf2e96a3c45b38f73a4188a6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:23:49 -0400 Subject: [PATCH 35/42] feat(apollo): render the reconciliation report Designed for the real asymmetry: 457 routine updates against 2 renames and a handful of decisions that need a human. The rows that matter must not be buried by the rows that do not. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Report.pm | 365 +++++++++++++++++++++++++ Model/t/report.t | 228 +++++++++++++++ 2 files changed, 593 insertions(+) create mode 100644 Model/lib/perl/ApolloRelease/Report.pm create mode 100644 Model/t/report.t diff --git a/Model/lib/perl/ApolloRelease/Report.pm b/Model/lib/perl/ApolloRelease/Report.pm new file mode 100644 index 0000000000..78f5ba2cb6 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Report.pm @@ -0,0 +1,365 @@ +package ApiCommonModel::Model::ApolloRelease::Report; + +use strict; +use warnings; + +# The artifact a release engineer reads, and then takes to the curation team to +# approve adds and prunes. It is the ONLY place a human sees what the tool +# intends to do before anything is generated or run, so it is rendered from the +# reconciliation result itself rather than reconstructed later from the command +# files -- an unapproved add or prune never reaches those files at all +# (Commands::writeCommandFiles), and it is exactly those that need reading. +# +# Pure: data in, string out. No I/O, no GUS_HOME, no database. The CLI writes +# the return value to report.txt and to stdout. +# +# LAYOUT IS THE POINT. A real build reconciles to roughly +# +# update 457 | add_candidate 35 | prune_candidate 2 | rename 2 +# exception 4 | redundant_overlay ~1 +# +# so 457 of the 500 rows need no reading at all, while the two renames are the +# rows whose loss costs curated annotations. Everything below follows from +# that asymmetry: decisions first, routine bulk last, and the one irreversible +# mistake -- unpublishing an organism that holds human annotations -- shouted +# rather than tabulated. + +# The marker a reader scans for, and the string the tests pin. One constant so +# a reword cannot silently apply to prunes but not to the summary. +my $ANNOTATED_PRUNE_WARNING = 'ANNOTATIONS WILL BE HIDDEN'; + +my @BUCKETS = qw(update add_candidate prune_candidate rename exception redundant_overlay); + +my $WIDTH = 78; + +# What still needs a human. Separated from render() because the CLI decides +# exit status and a caller should not have to grep the prose to learn whether +# the release is blocked. Approval is the gate: an approved candidate is a +# decision already made, so it is NOT pending -- but an approved prune of an +# annotated organism is still counted as annotated work at risk, because the +# thing worth knowing there is what happens, not who agreed to it. +sub pendingDecisions { + my ($class, $result) = @_; + + my @adds = grep { !$_->{approved} } @{$result->{add_candidate} || []}; + my @prunes = grep { !$_->{approved} } @{$result->{prune_candidate} || []}; + + return { + add => scalar @adds, + prune => scalar @prunes, + total => scalar(@adds) + scalar(@prunes), + annotated_prune => scalar(grep { _count($_->{annotation_count}) } @prunes), + }; +} + +sub render { + my ($class, $result, $context) = @_; + + $context ||= {}; + + my @out; + + push @out, "Apollo release reconciliation\n"; + push @out, sprintf("build %s, environment %s\n\n", + _or($context->{build}, '?'), _or($context->{environment}, '?')); + + # Every bucket is listed even at zero. A bucket that vanishes when empty + # cannot be distinguished from a bucket the report forgot, and "0 renames" is + # a fact a reader needs -- last release had two. + push @out, "summary\n"; + push @out, sprintf(" %-20s %5d\n", $_, scalar @{$result->{$_} || []}) for @BUCKETS; + push @out, "\n"; + + push @out, $class->_decisionBanner($result); + push @out, $class->_renames($result); + push @out, $class->_prunes($result); + push @out, $class->_adds($result); + push @out, $class->_exceptions($result); + push @out, $class->_redundantOverlay($result); + push @out, $class->_updates($result); + + return join('', @out); +} + +# Always emitted, including when nothing is pending. This is the line the +# release engineer reads first and the one they quote in the ticket; rendering +# it only when there is something to say makes its absence ambiguous between +# "nothing to approve" and "this report predates the check". +sub _decisionBanner { + my ($class, $result) = @_; + + my $pending = $class->pendingDecisions($result); + + unless ($pending->{total}) { + return "DECISIONS REQUIRED: none -- nothing is waiting on a human.\n\n"; + } + + my @out; + push @out, sprintf("DECISIONS REQUIRED: %d (%d add, %d prune)\n", + $pending->{total}, $pending->{add}, $pending->{prune}); + push @out, sprintf(" %d of the pending prune(s) would hide human annotations -- see below.\n", + $pending->{annotated_prune}) + if $pending->{annotated_prune}; + push @out, " Nothing is generated for an unapproved candidate; approve it in the\n"; + push @out, " roster overlay and re-run.\n\n"; + + return @out; +} + +# Two rows among five hundred, and the only ones where getting it wrong loses +# work that cannot be regenerated. Printed before anything else with content, +# with both abbrevs and the annotation count being preserved on one line, so +# the pair can be read back to a curator without cross-referencing. +sub _renames { + my ($class, $result) = @_; + + my @renames = sort { $a->{from_abbrev} cmp $b->{from_abbrev} } @{$result->{rename} || []}; + return () unless @renames; + + my @out = ("renames -- the existing Apollo organism is repointed in place;\n", + " its id, and every annotation on it, are preserved\n"); + foreach my $r (@renames) { + push @out, sprintf(" %-24s -> %-24s %4d annotation(s) %s\n", + $r->{from_abbrev}, $r->{to_abbrev}, + _count($r->{annotation_count}), + _name($r->{organism})); + } + push @out, "\n"; + + return @out; +} + +# The expensive mistake. A prune unpublishes an organism; if that organism +# holds human annotations they stop being visible, and nobody notices until a +# curator goes looking for their own work. The count is printed for every +# prune and the warning only for a non-zero one -- a marker on every line is a +# marker nobody reads. +sub _prunes { + my ($class, $result) = @_; + + my @prunes = sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{prune_candidate} || []}; + return () unless @prunes; + + my @out = ("prune candidates -- gone from the portal; unpublish only, reversible.\n", + " An UNAPPROVED line needs a decision.\n"); + foreach my $p (@prunes) { + my $count = _count($p->{annotation_count}); + push @out, sprintf(" %s %-24s %4d annotation(s) %s%s%s\n", + _box($p->{approved}), $p->{abbrev}, $count, + _or($p->{common_name}, ''), + _approvalNote($p), + $count ? " <-- $ANNOTATED_PRUNE_WARNING" : ''); + } + push @out, "\n"; + + return @out; +} + +sub _adds { + my ($class, $result) = @_; + + my @adds = sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{add_candidate} || []}; + return () unless @adds; + + my @out = ("add candidates -- reference+annotated on the portal, absent from Apollo.\n", + " An UNAPPROVED line needs a decision.\n"); + foreach my $a (@adds) { + push @out, sprintf(" %s %-24s %s%s\n", + _box($a->{approved}), $a->{abbrev}, + _name($a->{organism}), _approvalNote($a)); + } + push @out, "\n"; + + return @out; +} + +# No action, and said so on the section header, because the natural reading of +# "in Apollo but not reference+annotated" is that something is broken. It is +# not: it is a curation decision made once. The criterion each one fails is +# printed so a reader can tell a demoted reference strain from an unannotated +# one without going back to the portal. +sub _exceptions { + my ($class, $result) = @_; + + my @exceptions = sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{exception} || []}; + return () unless @exceptions; + + my @out = ("exceptions -- in Apollo but not reference+annotated. NO ACTION is taken\n", + " or needed; these are deliberate, do not \"fix\" them.\n"); + foreach my $e (@exceptions) { + push @out, sprintf(" %-24s %-28s %4d annotation(s) %s\n", + $e->{abbrev}, _failedCriteria($e), + _count($e->{annotation_count}), _name($e->{organism})); + } + push @out, "\n"; + + return @out; +} + +sub _failedCriteria { + my ($entry) = @_; + + my @failed; + push @failed, 'not reference' unless $entry->{is_reference}; + push @failed, 'not annotated' unless $entry->{is_annotated}; + + # Reachable only if a caller hands us an entry that does qualify; say so + # rather than printing an empty column that reads as "no reason given". + return 'criteria unknown' unless @failed; + + return join(', ', @failed); +} + +# An overlay line that no longer does anything is not an error -- it is a human +# decision overtaken by events. Printing it with its note is how the file gets +# tidied instead of accreting entries nobody dares delete. +sub _redundantOverlay { + my ($class, $result) = @_; + + my @entries = sort { $a->{abbrev} cmp $b->{abbrev} } @{$result->{redundant_overlay} || []}; + return () unless @entries; + + my @out = ("redundant overlay entries -- these lines no longer have any effect\n", + " and can be deleted from the overlay\n"); + foreach my $e (@entries) { + push @out, sprintf(" %-24s %-8s %s%s\n", + $e->{abbrev}, _or($e->{directive}, '?'), _or($e->{note}, ''), + defined $e->{reason} && length $e->{reason} + ? " (was: $e->{reason})" : ''); + } + push @out, "\n"; + + return @out; +} + +# LAST, and compressed. The judgement call in this module: 457 routine updates +# are printed, but as a wrapped list of abbrevs after every decision, not as +# 457 lines. +# +# Printing them one per line would make the update bucket ninety percent of the +# report and push the two renames off the first screen -- the exact failure the +# layout exists to avoid. Omitting them entirely is worse than it looks: the +# question actually asked of this section is "the count moved by three since +# last build, which three?", and a bare count cannot answer it. Wrapped, they +# cost about fifty lines at the very end, where a reader who has already found +# what they came for simply stops. +# +# The wrapped form is deliberately NOT the diffing surface -- inserting one +# abbrev reflows every later line. renderTsv() is where a release-to-release +# diff is taken, one record per line. +sub _updates { + my ($class, $result) = @_; + + my @updates = sort map { $_->{abbrev} } @{$result->{update} || []}; + return () unless @updates; + + my @out = (sprintf("routine updates -- %d organism(s) present in both Apollo and the\n", + scalar @updates), + " portal; regenerated and repointed, no decision needed.\n", + " Listed for diffing against the previous release only.\n"); + push @out, map { " $_\n" } _wrap(\@updates, $WIDTH - 2); + push @out, "\n"; + + return @out; +} + +sub _wrap { + my ($items, $width) = @_; + + my @lines; + my $line = ''; + foreach my $item (@$items) { + my $piece = length($line) ? " $item" : $item; + if (length($line) + length($piece) > $width) { + push @lines, $line; + $line = $item; + } + else { + $line .= $piece; + } + } + push @lines, $line if length $line; + + return @lines; +} + +# The machine-readable form, for diffing one release against the next. TSV +# rather than JSON because a diff is taken line by line: one record per line +# means an added organism is one added line, where a pretty-printed JSON object +# is several and a compact one is the whole file. Columns are fixed and the +# rows are sorted, so two runs over the same reconciliation are byte-identical +# and `diff` shows decisions, not formatting. +# +# One column carries two meanings (a rename's destination, an overlay line's +# directive) rather than widening every row with columns that are empty for +# five of the six buckets; the header says so. +my @COLUMNS = qw(bucket abbrev to_abbrev_or_directive apollo_id + annotation_count approved name note); + +sub renderTsv { + my ($class, $result) = @_; + + my @out = ('#' . join("\t", @COLUMNS) . "\n"); + + foreach my $bucket (@BUCKETS) { + my @rows = map { _tsvRow($bucket, $_) } @{$result->{$bucket} || []}; + push @out, map { join("\t", map { _clean($_) } @$_) . "\n" } + sort { $a->[1] cmp $b->[1] || $a->[2] cmp $b->[2] } @rows; + } + + return join('', @out); +} + +sub _tsvRow { + my ($bucket, $e) = @_; + + my %row = ( + bucket => $bucket, + abbrev => _or($e->{abbrev}, _or($e->{from_abbrev}, '')), + to_abbrev_or_directive => _or($e->{to_abbrev}, _or($e->{directive}, '')), + apollo_id => _or($e->{apollo_id}, ''), + annotation_count => defined $e->{annotation_count} ? _count($e->{annotation_count}) : '', + approved => defined $e->{approved} ? ($e->{approved} ? 1 : 0) : '', + name => _or($e->{common_name}, _name($e->{organism})), + note => _or($e->{note}, _or($e->{reason}, '')), + ); + + $row{note} = _failedCriteria($e) if $bucket eq 'exception'; + + return [map { $row{$_} } @COLUMNS]; +} + +# A tab or a newline reaching a TSV field would shift every later column, or +# split one record into two, in whatever reads the file. Portal common names +# are free text from a database, so this is reachable without malice. +sub _clean { + my ($value) = @_; + $value = '' unless defined $value; + $value =~ s/[\t\r\n]+/ /g; + return $value; +} + +sub _box { return $_[0] ? '[x]' : '[ ]' } + +sub _count { my ($n) = @_; return (defined $n && $n =~ /^\d+$/) ? $n + 0 : 0 } + +sub _or { + my ($value, $fallback) = @_; + return (defined $value && length $value) ? $value : $fallback; +} + +sub _name { + my ($organism) = @_; + return '' unless $organism; + return _or($organism->{name}, ''); +} + +sub _approvalNote { + my ($entry) = @_; + return '' unless $entry->{approved}; + return defined $entry->{reason} && length $entry->{reason} + ? " (approved in overlay: $entry->{reason})" + : ' (approved in overlay)'; +} + +1; diff --git a/Model/t/report.t b/Model/t/report.t new file mode 100644 index 0000000000..f068dd4c44 --- /dev/null +++ b/Model/t/report.t @@ -0,0 +1,228 @@ +use strict; +use warnings; +use Test::More; +use lib $ENV{GUS_HOME} . "/lib/perl"; +use ApiCommonModel::Model::ApolloRelease::Report; + +my $R = 'ApiCommonModel::Model::ApolloRelease::Report'; + +# A mixed result shaped like a real build: a large routine `update` bucket, two +# renames, an annotated prune next to a harmless one, an approved add next to an +# unapproved one, two exceptions failing on different criteria, and one dead +# overlay line. +sub mixedResult { + return { + update => [ + map { { abbrev => sprintf('org%03d', $_), + apollo_id => 1000 + $_, + public_mode => 1, + organism => {name => "Organism $_"} } } (1 .. 457) + ], + add_candidate => [ + { abbrev => 'hcapNAm1', + organism => {name => 'Histoplasma mississippiense NAm1'}, + approved => 0, reason => undef }, + { abbrev => 'pbraCM01', + organism => {name => 'Paracoccidioides brasiliensis CM01'}, + approved => 1, reason => 'approved by curation 2026-07' }, + ], + prune_candidate => [ + { abbrev => 'cglaCBS138', apollo_id => 21, common_name => 'Candida glabrata CBS 138', + annotation_count => 0, approved => 1, reason => 'renamed away, zero annotations' }, + { abbrev => 'zzzRisky', apollo_id => 22, common_name => 'Something Curated', + annotation_count => 14, approved => 0, reason => undef }, + ], + rename => [ + { from_abbrev => 'cneoJEC21', to_abbrev => 'cdenJEC21', apollo_id => 2452162, + annotation_count => 14, public_mode => 1, + organism => {name => 'Cryptococcus deneoformans JEC21'} }, + { from_abbrev => 'oldAbbrev', to_abbrev => 'newAbbrev', apollo_id => 31, + annotation_count => 0, public_mode => 0, + organism => {name => 'Renamed Thing'} }, + ], + exception => [ + { abbrev => 'tbruLister427_2018', apollo_id => 41, annotation_count => 3, + is_reference => 0, is_annotated => 1, public_mode => 1, + organism => {name => 'Trypanosoma brucei Lister427 2018'} }, + { abbrev => 'unannotatedRef', apollo_id => 42, annotation_count => 0, + is_reference => 1, is_annotated => 0, public_mode => 0, + organism => {name => 'Unannotated Reference'} }, + ], + redundant_overlay => [ + { abbrev => 'staleEntry', directive => 'add', + reason => 'approved 2025', note => 'already in Apollo' }, + ], + }; +} + +my $result = mixedResult(); +my $text = $R->render($result, {build => 71, environment => 'prod'}); + +# ------------------------------------------------------------------ counts + +like($text, qr/^\s*update\s+457\s*$/m, 'summary counts the update bucket'); +like($text, qr/^\s*add_candidate\s+2\s*$/m, 'summary counts the add bucket'); +like($text, qr/^\s*prune_candidate\s+2\s*$/m, 'summary counts the prune bucket'); +like($text, qr/^\s*rename\s+2\s*$/m, 'summary counts the rename bucket'); +like($text, qr/^\s*exception\s+2\s*$/m, 'summary counts the exception bucket'); +like($text, qr/^\s*redundant_overlay\s+1\s*$/m, 'summary counts the redundant overlay bucket'); + +like($text, qr/build 71/, 'the report says which build it describes'); +like($text, qr/\bprod\b/, 'and which environment'); + +# ----------------------------------------------------------------- renames +# Two rows out of 461 that must not be missed. + +like($text, qr/cneoJEC21.*cdenJEC21/, 'a rename names both abbrevs'); +like($text, qr/cneoJEC21.*cdenJEC21.*14/, 'and the annotation count it is preserving'); + +# ---------------------------------------------------- prunes: the loud one +# The expensive mistake this whole tool exists to prevent. + +like($text, qr/zzzRisky.*\b14\b/, 'a prune candidate shows its annotation count'); +like($text, qr/zzzRisky.*ANNOTATIONS WILL BE HIDDEN/, 'an annotated prune is flagged loudly'); +unlike($text, qr/cglaCBS138.*ANNOTATIONS WILL BE HIDDEN/, + 'a zero-annotation prune is NOT flagged'); + +# A prune with no annotation_count key at all must read as zero, not as a +# warning and not as an undef-warning in the middle of the report. +my $noCount = { %{mixedResult()}, + prune_candidate => [{abbrev => 'noCountAtAll', apollo_id => 9, + common_name => 'No Count', approved => 0}] }; +my $noCountText = $R->render($noCount, {build => 71, environment => 'prod'}); +like($noCountText, qr/noCountAtAll\s+0\b/, 'a missing annotation count reads as 0'); +unlike($noCountText, qr/noCountAtAll.*ANNOTATIONS WILL BE HIDDEN/, + 'and is not flagged'); + +# --------------------------------------------------------- approval status + +like($text, qr/hcapNAm1.*Histoplasma mississippiense NAm1/, 'an add candidate shows its organism name'); + +my ($unapprovedAdd) = $text =~ /^(.*hcapNAm1.*)$/m; +my ($approvedAdd) = $text =~ /^(.*pbraCM01.*)$/m; +like($approvedAdd, qr/approved/i, 'an add already approved in the overlay says so'); +unlike($unapprovedAdd, qr/approved/i, 'an unapproved add does not'); + +my ($approvedPrune) = $text =~ /^(.*cglaCBS138.*)$/m; +my ($unapprovedPrune) = $text =~ /^(.*zzzRisky.*)$/m; +like($approvedPrune, qr/approved/i, 'an approved prune says so'); +unlike($unapprovedPrune, qr/approved/i, 'an unapproved prune does not'); + +# -------------------------------------------------------------- exceptions + +my ($refException) = $text =~ /^(.*tbruLister427_2018.*)$/m; +my ($annException) = $text =~ /^(.*unannotatedRef.*)$/m; +like($refException, qr/not reference/i, 'an exception shows the criterion it fails'); +unlike($refException, qr/not annotated/i, 'and not the one it passes'); +like($annException, qr/not annotated/i, 'the other criterion is reported when it is the failing one'); +like($text, qr/exceptions.*no action/is, 'exceptions are marked as needing no action'); + +# ------------------------------------------------------- redundant overlay + +like($text, qr/staleEntry.*already in Apollo/, 'a redundant overlay entry is shown with its note'); +like($text, qr/staleEntry.*add\b/, 'and with the directive that is dead'); + +# -------------------------------------------------- pending human decisions + +# One unapproved add + one unapproved prune. +my $pending = $R->pendingDecisions($result); +is($pending->{add}, 1, 'pendingDecisions counts unapproved adds'); +is($pending->{prune}, 1, 'pendingDecisions counts unapproved prunes'); +is($pending->{total}, 2, 'and their total'); +is($pending->{annotated_prune}, 1, 'and how many pending prunes would hide annotations'); + +like($text, qr/DECISIONS REQUIRED:\s*2\b/, 'the report states the pending decision count plainly'); + +# An empty decision set must SAY so. Silence is indistinguishable from a +# report that forgot to render the section. +my $settled = { %{mixedResult()}, add_candidate => [], prune_candidate => [] }; +my $settledText = $R->render($settled, {build => 71, environment => 'prod'}); +like($settledText, qr/DECISIONS REQUIRED:\s*none/i, + 'a report with nothing pending says so explicitly rather than omitting the line'); + +my $settledPending = $R->pendingDecisions($settled); +is($settledPending->{total}, 0, 'and pendingDecisions agrees'); + +# Approved-but-annotated is still counted as annotated work at risk, but is not +# a pending decision -- the human already made it. +my $approvedRisky = { %{mixedResult()}, + add_candidate => [], + prune_candidate => [{abbrev => 'zzzRisky', apollo_id => 22, common_name => 'Something Curated', + annotation_count => 14, approved => 1, reason => 'signed off'}] }; +is($R->pendingDecisions($approvedRisky)->{total}, 0, 'an approved prune is not a pending decision'); +like($R->render($approvedRisky, {}), qr/zzzRisky.*ANNOTATIONS WILL BE HIDDEN/, + 'but it is still flagged: approval does not make the annotations less hidden'); + +# ------------------------------------------------ the 457-row update bucket +# Routine updates are real output -- "which 457" matters when the count moves -- +# but they must never sit between a reader and the rows that need a decision. + +my $pruneAt = index($text, 'zzzRisky'); +my $renameAt = index($text, 'cneoJEC21'); +my $addAt = index($text, 'hcapNAm1'); +my $updateAt = index($text, 'org001'); +cmp_ok($updateAt, '>', $pruneAt, 'routine updates are printed after the prune candidates'); +cmp_ok($updateAt, '>', $renameAt, 'and after the renames'); +cmp_ok($updateAt, '>', $addAt, 'and after the add candidates'); + +my @updateLines = grep { /\borg\d\d\d\b/ } split(/\n/, $text); +cmp_ok(scalar(@updateLines), '<', 100, + '457 updates do not become 457 lines that bury everything else'); +like($text, qr/org457/, 'but every updated organism is still named somewhere'); + +# ------------------------------------------------------------- determinism + +is($R->render(mixedResult(), {build => 71, environment => 'prod'}), + $R->render(mixedResult(), {build => 71, environment => 'prod'}), + 'the same input renders byte-identical output'); + +# Bucket contents arriving in a different order must not change the output; +# a diff between two releases has to be a diff of decisions. +my $shuffled = mixedResult(); +$shuffled->{prune_candidate} = [reverse @{$shuffled->{prune_candidate}}]; +$shuffled->{rename} = [reverse @{$shuffled->{rename}}]; +$shuffled->{add_candidate} = [reverse @{$shuffled->{add_candidate}}]; +$shuffled->{update} = [reverse @{$shuffled->{update}}]; +is($R->render($shuffled, {build => 71, environment => 'prod'}), $text, + 'input order does not change the rendered output'); + +# --------------------------------------------------------- missing buckets + +my $empty; +eval { $empty = $R->render({}, {}) }; +is($@, '', 'a result with no buckets at all renders instead of dying'); +like($empty, qr/^\s*update\s+0\s*$/m, 'and still shows a zero for every bucket'); + +# ------------------------------------------------------- machine-readable + +my $tsv = $R->renderTsv($result); +my @rows = split(/\n/, $tsv); +like($rows[0], qr/^#/, 'the TSV starts with a comment header naming the columns'); + +my @body = grep { !/^#/ } @rows; +is(scalar(@body), 457 + 2 + 2 + 2 + 2 + 1, 'the TSV has exactly one row per bucket entry'); + +my ($tsvRename) = grep { /\bcneoJEC21\b/ } @body; +my @f = split(/\t/, $tsvRename, -1); +is($f[0], 'rename', 'the first TSV column is the bucket'); +is($f[1], 'cneoJEC21', 'the second is the abbrev the entry is keyed by'); +like($tsvRename, qr/\bcdenJEC21\b/, 'a rename row carries the destination abbrev too'); +like($tsvRename, qr/\b14\b/, 'and the annotation count'); + +is($R->renderTsv(mixedResult()), $tsv, 'the TSV is byte-stable for the same input'); + +# A tab or newline in a portal-supplied name would silently shift every later +# column, or split one record into two, in whatever diffs this file. +my $nasty = { %{mixedResult()}, + prune_candidate => [{abbrev => 'nasty', apollo_id => 1, annotation_count => 0, approved => 0, + common_name => "Tab\there and\nnewline"}] }; +my ($nastyRow) = grep { /\bnasty\b/ } split(/\n/, $R->renderTsv($nasty)); +unlike($nastyRow, qr/\tTab\t/, 'an embedded tab is neutralised rather than shifting the columns'); + +my @nastyBody = grep { !/^#/ } split(/\n/, $R->renderTsv($nasty)); +is(scalar(@nastyBody), 457 + 2 + 1 + 2 + 2 + 1, + 'an embedded newline does not split one record into two rows'); +is(scalar(grep { !/^(update|add_candidate|prune_candidate|rename|exception|redundant_overlay)\t/ } @nastyBody), + 0, 'and every row still begins with its bucket'); + +done_testing(); From 67427129fcced3a51435bbb49846906a08b1ce31 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:25:49 -0400 Subject: [PATCH 36/42] test(apollo): pin the organism-name quoting guards Strain and isolate names carry punctuation and the portal is the source of that string, so an apostrophe reaching a --data line is reachable, not hypothetical. Pins that a single quote, a double quote and a backslash each refuse to emit, that the error names the organism and shows the offending character, and that the shell-quoting and JSON diagnoses are distinguishable. Also pins that ordinary punctuation is still accepted, so the guard cannot widen into general distrust. Co-Authored-By: Claude Opus 5 --- Model/t/commands.t | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/Model/t/commands.t b/Model/t/commands.t index 870a14484e..b76c01f360 100644 --- a/Model/t/commands.t +++ b/Model/t/commands.t @@ -150,6 +150,74 @@ for my $pair (['update', $update], ['rename', $rename], ['prune', $prune], ['add like($update, qr/"username":"admin\@local\.host"/, 'the admin user is admin@local.host'); +# --------------------------------------------- punctuation in an organism name + +# Strain and isolate names carry punctuation, and the portal is the source of +# that string. An apostrophe reaching the emitted line would terminate the +# shell's quoting of --data and turn the rest of the JSON into shell words; a +# double quote or a backslash would break the JSON instead. Both must refuse +# to emit rather than produce a line whose meaning depends on the shell. +# +# Asserted as facts, not prose: it dies, the message shows the offending value +# so a reader can see WHICH organism, and it shows the offending character. +sub dieFor { + my ($code) = @_; + eval { $code->(); 1 }; + return $@; +} + +sub renameNamed { + my ($name) = @_; + return sub { + $C->renameCommand({ + from_abbrev => 'lspGHANA_old', to_abbrev => 'lspGHANA', apollo_id => 3311, + public_mode => 1, organism => {name => $name, latest_annotation_version => 'Jan 2024'}, + }); + }; +} + +my $quotedName = q{Leishmania sp. 'ghana'}; +my $squote = dieFor(renameNamed($quotedName)); +ok($squote, 'a single quote in a rename commonName refuses to emit'); +like($squote, qr/\Q$quotedName\E/, 'and the error shows the offending organism name'); +like($squote, qr/'/, 'and the offending character itself'); + +my $dquoteName = q{Leishmania sp. "ghana"}; +my $dquote = dieFor(renameNamed($dquoteName)); +ok($dquote, 'a double quote in a rename commonName refuses to emit'); +like($dquote, qr/\Q$dquoteName\E/, 'and the error shows the offending organism name'); +like($dquote, qr/"/, 'and the offending character itself'); + +my $slashName = q{Leishmania sp. \ghana}; +my $slash = dieFor(renameNamed($slashName)); +ok($slash, 'a backslash in a rename commonName refuses to emit'); +like($slash, qr/\Q$slashName\E/, 'and the error shows the offending organism name'); +like($slash, qr/\\/, 'and the offending character itself'); + +# The shell-quoting failure and the JSON failure are different problems with +# different fixes, so a reader must be able to tell which one they hit. +isnt($squote, $dquote, 'the single-quote and double-quote diagnoses differ'); + +# The same hazard reaches addCommand by its own path -- the name is +# shell-quoted there rather than embedded in JSON, so it has its own guard. +my $addQuoted = dieFor(sub { + $C->addCommand({abbrev => 'lspGHANA', approved => 1, + organism => {name => $quotedName, latest_annotation_version => 'Jan 2024'}}); +}); +ok($addQuoted, 'a single quote in an added organism name refuses to emit'); +like($addQuoted, qr/\Q$quotedName\E/, 'and the add error shows the offending organism name'); +like($addQuoted, qr/'/, 'and the offending character itself'); + +# A clean name with other punctuation must still go through: the guard is about +# three specific characters, not a general distrust of the portal. +my $punctuated = $C->addCommand({ + abbrev => 'psp_G1', approved => 1, + organism => {name => 'Plasmodium sp. gorilla clade G1 (strain-2)', + latest_annotation_version => 'Jan 2024'}, +}); +like($punctuated, qr/-name 'Plasmodium sp\. gorilla clade G1 \(strain-2\)/, + 'ordinary punctuation in a name is not rejected'); + # ---------------------------------------------------- writing the two files my $organism = sub { From eb60a595c1c2da0f1a6dc3ca29f762be1c620a20 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:35:41 -0400 Subject: [PATCH 37/42] fix(apollo): count every annotated prune, not just the unapproved ones pendingDecisions derived annotated_prune from the already-filtered unapproved list, so the field disagreed with the report it summarises: the text flagged two organisms losing annotations while a caller reading the field was told one. Approval gates the action, not the consequence. The banner warning moves out of the pending branch for the same reason -- an approved annotated prune is exactly the case where the release looks ready to run unread. Co-Authored-By: Claude Opus 5 --- Model/lib/perl/ApolloRelease/Report.pm | 53 ++++++++++++++++-------- Model/t/report.t | 56 +++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 18 deletions(-) diff --git a/Model/lib/perl/ApolloRelease/Report.pm b/Model/lib/perl/ApolloRelease/Report.pm index 78f5ba2cb6..53405539e3 100644 --- a/Model/lib/perl/ApolloRelease/Report.pm +++ b/Model/lib/perl/ApolloRelease/Report.pm @@ -34,21 +34,33 @@ my $WIDTH = 78; # What still needs a human. Separated from render() because the CLI decides # exit status and a caller should not have to grep the prose to learn whether -# the release is blocked. Approval is the gate: an approved candidate is a -# decision already made, so it is NOT pending -- but an approved prune of an -# annotated organism is still counted as annotated work at risk, because the -# thing worth knowing there is what happens, not who agreed to it. +# the release is blocked. +# +# THREE of these four fields count PENDING decisions; the fourth does not, and +# the difference is load-bearing. Approval is a gate on an ACTION: an approved +# candidate is a decision already made, so it is not pending. It is not a gate +# on a CONSEQUENCE -- approving a prune does not make the annotations it hides +# any less hidden. So `annotated_prune` counts EVERY prune candidate that +# holds annotations, approved or not, which is exactly the set _prunes() flags +# with ANNOTATIONS WILL BE HIDDEN in the text. +# +# Deriving it from the unapproved subset was the original bug here, and it is +# the quiet kind: the prose says two organisms lose their annotations while the +# field a caller reads says one. Reconciling those two answers is the entire +# reason this method exists rather than leaving the CLI to grep the report. sub pendingDecisions { my ($class, $result) = @_; - my @adds = grep { !$_->{approved} } @{$result->{add_candidate} || []}; - my @prunes = grep { !$_->{approved} } @{$result->{prune_candidate} || []}; + my @allPrunes = @{$result->{prune_candidate} || []}; + + my @adds = grep { !$_->{approved} } @{$result->{add_candidate} || []}; + my @prunes = grep { !$_->{approved} } @allPrunes; return { add => scalar @adds, prune => scalar @prunes, total => scalar(@adds) + scalar(@prunes), - annotated_prune => scalar(grep { _count($_->{annotation_count}) } @prunes), + annotated_prune => scalar(grep { _count($_->{annotation_count}) } @allPrunes), }; } @@ -90,18 +102,27 @@ sub _decisionBanner { my $pending = $class->pendingDecisions($result); - unless ($pending->{total}) { - return "DECISIONS REQUIRED: none -- nothing is waiting on a human.\n\n"; - } - my @out; - push @out, sprintf("DECISIONS REQUIRED: %d (%d add, %d prune)\n", - $pending->{total}, $pending->{add}, $pending->{prune}); - push @out, sprintf(" %d of the pending prune(s) would hide human annotations -- see below.\n", + + push @out, $pending->{total} + ? sprintf("DECISIONS REQUIRED: %d (%d add, %d prune)\n", + $pending->{total}, $pending->{add}, $pending->{prune}) + : "DECISIONS REQUIRED: none -- nothing is waiting on a human.\n"; + + # Deliberately OUTSIDE the pending branch, and above the procedural note. An + # annotated prune a curator already approved is still an organism whose + # annotations are about to stop being visible; printing this only when a + # decision happens to be outstanding would drop it in exactly the case the + # release otherwise looks ready to run unread. + push @out, sprintf(" %d prune candidate(s) here would hide human annotations -- see below.\n", $pending->{annotated_prune}) if $pending->{annotated_prune}; - push @out, " Nothing is generated for an unapproved candidate; approve it in the\n"; - push @out, " roster overlay and re-run.\n\n"; + + push @out, " Nothing is generated for an unapproved candidate; approve it in the\n", + " roster overlay and re-run.\n" + if $pending->{total}; + + push @out, "\n"; return @out; } diff --git a/Model/t/report.t b/Model/t/report.t index f068dd4c44..48871fb753 100644 --- a/Model/t/report.t +++ b/Model/t/report.t @@ -115,7 +115,13 @@ my ($annException) = $text =~ /^(.*unannotatedRef.*)$/m; like($refException, qr/not reference/i, 'an exception shows the criterion it fails'); unlike($refException, qr/not annotated/i, 'and not the one it passes'); like($annException, qr/not annotated/i, 'the other criterion is reported when it is the failing one'); -like($text, qr/exceptions.*no action/is, 'exceptions are marked as needing no action'); +# Anchored to the section header line rather than scanning the whole report +# with /s: the previous form would have passed on "no action" appearing +# anywhere below, in any later section. Two facts are pinned -- a header line +# beginning at column 0 with "exceptions" (the indented rows never do), and +# "no action" on that same line -- with no claim on the prose between them. +like($text, qr/^exceptions\b[^\n]*\bno action\b/mi, + 'the exceptions header itself says no action is taken'); # ------------------------------------------------------- redundant overlay @@ -129,7 +135,7 @@ my $pending = $R->pendingDecisions($result); is($pending->{add}, 1, 'pendingDecisions counts unapproved adds'); is($pending->{prune}, 1, 'pendingDecisions counts unapproved prunes'); is($pending->{total}, 2, 'and their total'); -is($pending->{annotated_prune}, 1, 'and how many pending prunes would hide annotations'); +is($pending->{annotated_prune}, 1, 'and how many prunes would hide annotations'); like($text, qr/DECISIONS REQUIRED:\s*2\b/, 'the report states the pending decision count plainly'); @@ -153,6 +159,52 @@ is($R->pendingDecisions($approvedRisky)->{total}, 0, 'an approved prune is not a like($R->render($approvedRisky, {}), qr/zzzRisky.*ANNOTATIONS WILL BE HIDDEN/, 'but it is still flagged: approval does not make the annotations less hidden'); +# The two halves of that principle, asserted TOGETHER, because they pull in +# opposite directions and the bug was assuming one gate governed both. +# Approval gates the ACTION (is a decision outstanding?) and NOT the +# CONSEQUENCE (do annotations stop being visible?). Every earlier fixture had +# its annotated prune unapproved, so both readings gave the same number and +# neither was actually pinned -- pendingDecisions counted 1 while the report +# flagged 2, and nothing was red. +my $mixedRisk = { %{mixedResult()}, + add_candidate => [], + prune_candidate => [ + {abbrev => 'approvedRisky', apollo_id => 1, common_name => 'Approved And Curated', + annotation_count => 20, approved => 1, reason => 'signed off by curation'}, + {abbrev => 'unapprovedRisky', apollo_id => 2, common_name => 'Pending And Curated', + annotation_count => 5, approved => 0}, + {abbrev => 'approvedHarmless', apollo_id => 3, common_name => 'Nothing To Lose', + annotation_count => 0, approved => 1, reason => 'empty, safe'}, + ] }; + +my $risk = $R->pendingDecisions($mixedRisk); +is($risk->{annotated_prune}, 2, + 'annotated_prune counts EVERY prune holding annotations, approved or not'); +is($risk->{prune}, 1, 'while prune counts only the one still awaiting a decision'); +is($risk->{total}, 1, 'and total follows the pending decision, not the risk'); + +# ...and the structured count must agree with what the text actually shows. +# This is the assertion the defect broke: prose said two, the field said one. +my $riskText = $R->render($mixedRisk, {build => 71, environment => 'prod'}); +my @flagged = grep { /ANNOTATIONS WILL BE HIDDEN/ } split(/\n/, $riskText); +is(scalar(@flagged), $risk->{annotated_prune}, + 'the flagged lines in the report and the annotated_prune field agree'); +like($riskText, qr/approvedRisky.*ANNOTATIONS WILL BE HIDDEN/, 'the approved one is flagged'); +like($riskText, qr/unapprovedRisky.*ANNOTATIONS WILL BE HIDDEN/, 'the unapproved one is flagged'); +unlike($riskText, qr/approvedHarmless.*ANNOTATIONS WILL BE HIDDEN/, 'and the empty one is not'); + +# The banner warning must survive a report with NO decisions pending -- that is +# precisely the run that looks ready to execute unread. +my $allApprovedRisk = { %{mixedResult()}, + add_candidate => [], + prune_candidate => [{abbrev => 'approvedRisky', apollo_id => 1, + common_name => 'Approved And Curated', + annotation_count => 20, approved => 1, reason => 'signed off'}] }; +my $allApprovedText = $R->render($allApprovedRisk, {build => 71, environment => 'prod'}); +like($allApprovedText, qr/DECISIONS REQUIRED:\s*none/i, 'nothing is pending'); +like($allApprovedText, qr/^\s*1 prune candidate\(s\).*hide human annotations/m, + 'yet the banner still says annotations will be hidden'); + # ------------------------------------------------ the 457-row update bucket # Routine updates are real output -- "which 457" matters when the count moves -- # but they must never sit between a reader and the rows that need a decision. From 0c4a7f648b414a99d0f10dbf6903faa3cf907a13 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 21 Aug 2026 23:39:21 -0400 Subject: [PATCH 38/42] feat(apollo): add the createApolloReleasePackage CLI Resolves renames from the database first and falls back to assembly identity, refuses absurd input rather than producing an empty release, and will not generate while human decisions are pending. Co-Authored-By: Claude Opus 5 --- Model/bin/createApolloReleasePackage | 263 ++++++++++++ Model/lib/perl/ApolloRelease/Cli.pm | 576 +++++++++++++++++++++++++++ Model/t/cli.t | 459 +++++++++++++++++++++ 3 files changed, 1298 insertions(+) create mode 100755 Model/bin/createApolloReleasePackage create mode 100644 Model/lib/perl/ApolloRelease/Cli.pm create mode 100644 Model/t/cli.t diff --git a/Model/bin/createApolloReleasePackage b/Model/bin/createApolloReleasePackage new file mode 100755 index 0000000000..8dec65eef7 --- /dev/null +++ b/Model/bin/createApolloReleasePackage @@ -0,0 +1,263 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +# --help must be answerable on a machine where nothing is configured, so the +# library path is only added when GUS_HOME is actually set. An unconditional +# "$ENV{GUS_HOME}/lib/perl" would emit an uninitialized warning at COMPILE time +# -- before any option has been looked at -- and the first thing a new user +# would see is Perl noise instead of the usage text. +use lib (defined $ENV{GUS_HOME} && length $ENV{GUS_HOME} + ? "$ENV{GUS_HOME}/lib/perl" : ()); + +use File::Path qw(make_path); + +# --------------------------------------------------------------------------- +# createApolloReleasePackage +# +# Wiring only. Every rule with a wrong answer worth catching lives in +# ApolloRelease::Cli or in the module that owns it, because a Perl script cannot +# be `use`d by a test without running its main(). The tool this replaces put +# its thresholds, its argument handling and its rename detection in a script in +# a home directory, and so nothing ever exercised them: it produced an empty +# release for build 71 and exited 0. +# +# Order below is deliberate and is the whole safety story: +# 1. parse options -- no environment, no database +# 2. preflight -- everything checkable before real work +# 3. read the three inputs -- portal, Apollo, overlay. All read-only. +# 4. sanity -- refuse absurd input rather than ship an +# empty release +# 5. resolve renames -- database first, assembly identity second +# 6. reconcile + report -- --report stops here, having changed nothing +# 7. gate -- no generation while a human decision is open +# 8. generate -- per-organism, failures isolated +# --------------------------------------------------------------------------- + +my $CLI = 'ApiCommonModel::Model::ApolloRelease::Cli'; + +# Parsed before anything is loaded from GUS_HOME: --help and a missing required +# option must not require credentials, a GUS_HOME or a database. +my $opt = eval { _requireCli(); $CLI->parseOptions(@ARGV) }; +_fail($@) unless $opt; + +if ($opt->{phase} eq 'help') { + print $CLI->usage(); + exit 0; +} + +# Everything from preflight onwards reports as one clean line rather than a Perl +# die with a file and line number: these messages are read by a release +# engineer, not by whoever wrote the module. +my $status = eval { main($opt) }; +_fail($@) unless defined $status; +exit $status; + +sub main { + my ($opt) = @_; + + # --- 2. preflight -------------------------------------------------------- + # + # Run in BOTH phases. --report is the rehearsal for --generate: a report that + # succeeds and is then followed by a --generate dying at startup on a missing + # tool or a mistyped config key wastes the very round trip to the curation + # team that the two-phase split exists to protect. + $CLI->assertEnvironment(\%ENV, !$opt->{apollo_roster}); + $CLI->assertPreviousRelease($opt->{previous_release}); + + my $outDir = $CLI->outputDir($opt); + + my %generateOpts = ( + outDir => "$outDir", + base => $opt->{base_url}, + project => $opt->{project}, + build => $opt->{build}, + wsDir => $opt->{ws_dir}, + gusHome => $ENV{GUS_HOME}, + ); + $CLI->assertGenerateConfig(\%generateOpts); + + _generate()->assertToolsAvailable(); + + # --- 3. inputs, all read-only ------------------------------------------- + my $portal = _portal()->loadFromCommand($opt->{project}); + $CLI->assertPortalSane($portal); + + my ($live, $apolloSource) = _loadApollo($opt); + $CLI->assertApolloSane($live, $apolloSource); + + my $overlayPath = "$ENV{GUS_HOME}/data/ApiCommonModel/Model/apollo/roster-overlay.txt"; + my $overlay = _overlay()->parseFile($overlayPath); + + # --- 5. renames ---------------------------------------------------------- + my $resolved = $CLI->resolveRenames($portal, $live, { + previous_release => $opt->{previous_release}, + ws_dir => $opt->{ws_dir}, + project => $opt->{project}, + build => $opt->{build}, + }); + + # --- 6. reconcile -------------------------------------------------------- + my $result = _reconcile()->reconcile($portal, $live, $overlay, $resolved->{renames}); + $CLI->assertUpdateBucketSane($result, $opt->{force}); + + # Portal.pm and the rename resolution COLLECT their warnings rather than + # writing to stderr -- their own callers treat a child's stderr byte as proof + # its output is untrustworthy. Surfacing them is therefore this script's job, + # and skipping it turns a handled skip into an invisible one. + _printWarnings('portal', [_portal()->warnings()]); + _printWarnings('renames', $resolved->{warnings}); + _printRenameProvenance($resolved); + + print "Apollo roster read from $apolloSource\n"; + print "roster overlay read from $overlayPath\n\n"; + + my $report = _report()->render($result, {build => $opt->{build}, + environment => $opt->{environment}}); + print $report; + + # Printed AFTER the report and independently of the decision gate. An + # annotated prune that curation has already approved raises no pending + # decision, so it is exactly the case that runs unread -- see + # Cli::annotatedPruneWarning. + my $atStake = $CLI->annotatedPruneWarning($result); + print "\nAT STAKE: $atStake" if $atStake; + + # --- --report stops here, having changed nothing ------------------------- + if ($opt->{phase} eq 'report') { + print "\n--report changed nothing. Re-run with --generate to build the package\n" + . "into $outDir once the decisions above are recorded in the roster overlay.\n"; + return 0; + } + + # --- 7. the gate --------------------------------------------------------- + $CLI->assertGenerationAllowed($result, $opt->{force}); + + # --- 8. generate --------------------------------------------------------- + my $roster = $CLI->generationRoster($result, $opt->{organisms}); + + if (@{$opt->{organisms}}) { + print "\nPARTIAL PACKAGE: --organism narrowed generation to " + . scalar(@$roster) . " organism(s).\n" + . "The report above covers the whole roster; the command files below cover\n" + . "only what was generated.\n"; + } + + my $commandDir = "$outDir/updateCommands"; + make_path("$outDir/data", "$outDir/twoBit", $commandDir); + + print "\ngenerating " . scalar(@$roster) . " organism(s) into $outDir\n"; + + my $generated = _generate()->generateAll($roster, sub { + my ($organism) = @_; + return _generate()->generateOrganism($organism, \%generateOpts); + }); + + _printWarnings('generate', [_generate()->warnings()]); + + # Written from the NARROWED result so the commands describe the package that + # was actually built. Full-roster commands beside a partial package would + # repoint an Apollo organism at a directory nobody generated. + my $written = _commands()->writeCommandFiles( + $CLI->narrowResult($result, $opt->{organisms}), $commandDir, + build => $opt->{build}); + + _write("$outDir/report.txt", $report); + _write("$outDir/report.tsv", _report()->renderTsv($result)); + + printf("\n%d succeeded, %d failed\n", + scalar @{$generated->{succeeded}}, scalar @{$generated->{failed}}); + print " $written->{curl}\n $written->{groovy}\n"; + print " $outDir/report.txt\n $outDir/report.tsv\n"; + + return 0 unless @{$generated->{failed}}; + + # Per-organism failures, printed in full: the run is hours long and the + # organisms are independent, so a failure is recorded and the loop continues. + # Exiting zero here is how a package ships with holes in it. + print STDERR "\nFAILED organisms:\n"; + print STDERR " $_: $generated->{errors}{$_}\n" for @{$generated->{failed}}; + + return 1; +} + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +sub _loadApollo { + my ($opt) = @_; + + # The API is IP-restricted to Penn hosts and needs credentials. A saved + # findAllOrganisms response goes through the same normalise() seam, so an + # offline rehearsal exercises every rule the live path does except the fetch. + return (_apollo()->loadFromFile($opt->{apollo_roster}), $opt->{apollo_roster}) + if $opt->{apollo_roster}; + + return (_apollo()->loadFromApi(), + $ENV{APOLLO_API_URL} || 'https://apollo-api.veupathdb.org'); +} + +sub _printWarnings { + my ($label, $warnings) = @_; + + return unless $warnings && @$warnings; + + print "\n" . scalar(@$warnings) . " $label warning(s):\n"; + print " $_\n" for @$warnings; + print "\n"; +} + +# A rename that was resolved silently is a rename nobody audited, and it is the +# one action that touches curated annotations. Say which mechanism decided it: +# "the database says so" and "the sequences match" carry very different weight. +sub _printRenameProvenance { + my ($resolved) = @_; + + my @from = sort keys %{$resolved->{renames}}; + return unless @from || @{$resolved->{unresolved}}; + + print "\nrename resolution\n"; + printf(" %-24s -> %-24s (%s)\n", + $_, $resolved->{renames}{$_}, $resolved->{mechanism}{$_}) for @from; + print " unresolved orphan(s), left as prune candidates: " + . join(', ', @{$resolved->{unresolved}}) . "\n" + if @{$resolved->{unresolved}}; + print "\n"; +} + +sub _write { + my ($path, $content) = @_; + open(my $fh, '>:raw', $path) or die "Cannot write $path: $!\n"; + print $fh $content; + close $fh or die "Cannot close $path: $!\n"; + return 1; +} + +sub _fail { + my ($error) = @_; + chomp(my $message = $error || 'failed for no stated reason'); + print STDERR "createApolloReleasePackage: $message\n"; + exit 2; +} + +# Loaded lazily and by name so that --help and an option error never touch +# GUS_HOME. Each wrapper returns the class name, so a caller reads as a normal +# method call. +sub _requireCli { _load('Cli') } +sub _portal { _load('Portal') } +sub _apollo { _load('Apollo') } +sub _overlay { _load('Overlay') } +sub _reconcile { _load('Reconcile') } +sub _report { _load('Report') } +sub _generate { _load('Generate') } +sub _commands { _load('Commands') } + +sub _load { + my ($name) = @_; + my $class = "ApiCommonModel::Model::ApolloRelease::$name"; + eval "require $class; 1" or die $@; + return $class; +} + diff --git a/Model/lib/perl/ApolloRelease/Cli.pm b/Model/lib/perl/ApolloRelease/Cli.pm new file mode 100644 index 0000000000..1b3f26cd98 --- /dev/null +++ b/Model/lib/perl/ApolloRelease/Cli.pm @@ -0,0 +1,576 @@ +package ApiCommonModel::Model::ApolloRelease::Cli; + +use strict; +use warnings; + +use Getopt::Long qw(GetOptionsFromArray); + +use ApiCommonModel::Model::ApolloRelease::Rename; +use ApiCommonModel::Model::ApolloRelease::Report; + +# Everything in createApolloReleasePackage that can be decided without a +# database, a subprocess or a filesystem lives here rather than in the script. +# +# A Perl script cannot be `use`d by a test without running its main(), so logic +# left in Model/bin/ is logic that is never exercised until a release engineer +# runs it against prod. That is precisely how the tool this replaces came to +# report success on an empty release: its argument handling, its thresholds and +# its rename detection had no seam anyone could test. So the script is a thin +# wiring layer over this module, and every rule with a wrong answer worth +# catching is a class method here. +# +# In particular the RENAME RESOLUTION lives here and not in Rename.pm. Rename.pm +# is finished and reviewed, and is about one thing: does this .fai describe the +# same assembly as that one. Resolving a rename is a two-mechanism policy -- +# consult apidb.organism first, fall back to assembly identity -- and the first +# mechanism needs no file I/O at all. Putting it in Rename.pm would give that +# module a second, unrelated reason to change. + +# --------------------------------------------------------------------------- +# Sanity floor for the portal organism count +# --------------------------------------------------------------------------- +# +# The portal returned 831 organisms on build 71, and an organism set only ever +# grows: an organism is retired by losing its reference/annotated flags, not by +# leaving the list. So any large drop means the query, the model or the project +# name is wrong, not that VEuPathDB shrank. +# +# 500 is the floor, chosen against the one number that makes an undercount +# dangerous rather than merely odd: live prod Apollo holds 459 organisms, and +# every Apollo organism absent from the portal becomes a prune candidate. Below +# ~459 the tool would be proposing to unpublish curated genomes on the strength +# of a broken query -- so the floor sits just above it. It is far enough below +# 831 (a 40% loss) that ordinary curation churn, or a component database being +# reloaded, can never trip it. A run that legitimately has fewer organisms than +# this does not exist today; when it does, this constant is the one place to +# argue about it. +use constant PORTAL_FLOOR => 500; + +my $RENAME = 'ApiCommonModel::Model::ApolloRelease::Rename'; +my $REPORT = 'ApiCommonModel::Model::ApolloRelease::Report'; + +my @ENVIRONMENTS = qw(qa prod); + +# --------------------------------------------------------------------------- +# Options +# --------------------------------------------------------------------------- + +sub usage { + return <<'USAGE'; +createApolloReleasePackage --build N (--report | --generate) [options] + +Builds the JBrowse configuration and sequence data that Apollo, the genome +curation platform, serves for a VEuPathDB release -- and the command files a +human then runs against Apollo. It never calls a mutating Apollo endpoint. + +Phases (exactly one, they cost differently): + --report minutes. Portal + Apollo + overlay + rename resolution + + reconciliation, printed. Changes nothing on disk. This is + what goes to the curation team. + --generate hours. Everything --report does, then builds the package for + the approved roster and writes the update command files. + +Options: + --build N release build number (required) + --environment qa|prod which roster to write (default prod) + --project NAME WDK model name (default UniDB) + --out-dir DIR package root (default $HOME/apolloConfigs) + --base-url URL absolutization base (default https://veupathdb.org) + --webservices-dir DIR webServices tree holding the genomes + (default /var/www/Common/apiSiteFilesMirror/webServices) + --previous-release DIR the previous release's directory, i.e. the one + containing data/. Enables the assembly-identity + fallback for renames the database cannot explain. + --organism ABBREV narrow GENERATION to this organism; repeatable. + Reconciliation always runs over everything, so the + report and the safety invariants are unaffected. + --apollo-roster FILE read the Apollo roster from a saved + findAllOrganisms response instead of the API. For + offline rehearsal; a real release uses the API. + --force proceed past the pending-decision gate and past an + empty update bucket. Neither is overridden lightly. + --help this text + +Output: /release-// + data// twoBit/.2bit updateCommands/ report.txt report.tsv + +Environment: GUS_HOME, and APOLLO_API_USER / APOLLO_API_PASS unless +--apollo-roster is given. No password is ever read from the source. +USAGE +} + +# Dies with a plain message (no Perl line noise) on any bad combination. Reads +# nothing but @argv and %ENV{HOME}: --help and a missing required option must be +# answerable with no credentials, no GUS_HOME and no database. +sub parseOptions { + my ($class, @argv) = @_; + + my %opt = ( + environment => 'prod', + project => 'UniDB', + out_dir => ($ENV{HOME} || '.') . '/apolloConfigs', + base_url => 'https://veupathdb.org', + ws_dir => '/var/www/Common/apiSiteFilesMirror/webServices', + organisms => [], + ); + + my ($report, $generate, $help); + + my $parser = Getopt::Long::Parser->new(config => ['no_auto_abbrev', 'no_ignore_case']); + + # GetOptionsFromArray warns to stderr and returns false. Turn that into the + # same kind of death as every other bad option, so a caller sees one message. + my $problem; + local $SIG{__WARN__} = sub { $problem ||= $_[0] }; + + $parser->getoptionsfromarray( + \@argv, + 'report' => \$report, + 'generate' => \$generate, + 'help' => \$help, + 'build=s' => \$opt{build}, + 'environment=s' => \$opt{environment}, + 'project=s' => \$opt{project}, + 'out-dir=s' => \$opt{out_dir}, + 'base-url=s' => \$opt{base_url}, + 'webservices-dir=s' => \$opt{ws_dir}, + 'previous-release=s' => \$opt{previous_release}, + 'apollo-roster=s' => \$opt{apollo_roster}, + 'organism=s' => $opt{organisms}, + 'force' => \$opt{force}, + ) or do { chomp(my $m = $problem || 'bad options'); die "$m\n" }; + + die "unexpected argument(s): @argv\n" if @argv; + + # --help short-circuits every other rule. It is the one invocation that must + # work on a machine where nothing is configured. + if ($help) { + $opt{phase} = 'help'; + return \%opt; + } + + die "exactly one of --report or --generate is required (they cost differently:\n" + . "--report takes minutes and changes nothing; --generate takes hours)\n" + unless ($report ? 1 : 0) + ($generate ? 1 : 0) == 1; + + $opt{phase} = $report ? 'report' : 'generate'; + + die "--build N is required (the release build number)\n" + unless defined $opt{build} && length $opt{build}; + die "--build must be a positive integer, got '$opt{build}'\n" + unless $opt{build} =~ /^[1-9][0-9]*$/; + $opt{build} += 0; + + die "--environment must be one of: @ENVIRONMENTS (got '$opt{environment}')\n" + unless grep { $_ eq $opt{environment} } @ENVIRONMENTS; + + # --organism narrows generation only. Accepting it on --report would produce + # a report that looks filtered and is not, which is worse than refusing it. + die "--organism narrows generation only and has no effect with --report;\n" + . "the reconciliation always runs over every organism.\n" + if @{$opt{organisms}} && $opt{phase} eq 'report'; + + foreach my $key (qw(project base_url ws_dir out_dir)) { + die "--" . ($key =~ s/_/-/gr) . " cannot be empty\n" + unless defined $opt{$key} && length $opt{$key}; + } + + return \%opt; +} + +sub outputDir { + my ($class, $opt) = @_; + return "$opt->{out_dir}/release-$opt->{build}/$opt->{environment}"; +} + +# --------------------------------------------------------------------------- +# Preflight -- everything checkable before any real work +# --------------------------------------------------------------------------- + +# $env is passed in rather than read from %ENV so this is testable, and so the +# script has one place that decides which variables a run needs. The Apollo +# credentials are genuinely not needed when the roster comes from a file. +sub assertEnvironment { + my ($class, $env, $needApolloCredentials) = @_; + + my @required = ('GUS_HOME'); + push @required, qw(APOLLO_API_USER APOLLO_API_PASS) if $needApolloCredentials; + + foreach my $name (@required) { + die "$name is not set.\n" + . "Source the site's etc/setenv, and export the Apollo API credentials\n" + . "(APOLLO_API_USER / APOLLO_API_PASS) -- they are never stored in the repo.\n" + unless defined $env->{$name} && length $env->{$name}; + } + + return 1; +} + +sub assertPreviousRelease { + my ($class, $dir) = @_; + + return 1 unless defined $dir && length $dir; + + die "--previous-release $dir does not exist\n" unless -e $dir; + die "--previous-release $dir is not a directory\n" unless -d $dir; + die "--previous-release $dir is not readable\n" unless -r $dir; + + return 1; +} + +# The keys Generate::generateOrganism reads out of its %$opts. A quality review +# flagged both failure modes this exists to stop: `wsdir` for `wsDir` fails deep +# into a run when the first genome cannot be found, and a missing `gusHome` +# never fails at all -- it degrades to running "/bin/