Add reproducible testing infrastructure

Uses testcontainers to spin up a reproducible test environment in a reasonable amount of time. 500 paths are sampled from real data to use for testing. These tests already exposed real bugs in the query code for counting and pathways.

* Do not format or lint the dries/ directory
* Added tests for the number of pathways from a specific tracer/gauge pair
* Added tests for counting
* Added tests for pathways counting
* Set required environment variables in github CI to dummy values
* Move the parse_literal function from submit to abomination
* parse_literal now returns None on error
* Generate tests based on sampled data
* Added a test for total data size
* Generate test data by randomly sampling real data
* Run tests and type-checking in parallel with linting/formatting
* Configure code coverage reporting
* Correct strict typing issues

Author: MaybeJustJames

.github/workflows/server-pr.yaml

@@ -6,10 +6,100 @@ on:
       - 'server/**'
 
 jobs:
-  code-quality:
+  type-check:
     runs-on: ubuntu-latest
     env:
-      working-directory: ./server
+      working-directory: ${{ github.workspace }}/server
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          virtualenvs-path: ${{ env.working-directory }}/.venv
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.working-directory }}/.venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock')}}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+        working-directory: ${{ env.working-directory }}
+
+      - name: Install project
+        if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true'
+        run: poetry install --no-interaction
+        working-directory: ${{ env.working-directory }}
+
+      - name: Type checking
+        run: |
+          source .venv/bin/activate
+          mypy --strict ttfd/
+        working-directory: ${{ env.working-directory }}
+
+  tests:
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ${{ github.workspace }}/server
+      CLIENT_ID: ""
+      CLIENT_SECRET: ""
+      DATABASE_URI: ""
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          virtualenvs-path: ${{ env.working-directory }}/.venv
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.working-directory }}/.venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock')}}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+        working-directory: ${{ env.working-directory }}
+
+      - name: Install project
+        if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true'
+        run: poetry install --no-interaction
+        working-directory: ${{ env.working-directory }}
+
+      - name: Run tests
+        run: |
+          source .venv/bin/activate
+          coverage run -m pytest
+          coverage report
+        working-directory: ${{ env.working-directory }}
+
+  lint-format:
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ${{ github.workspace }}/server
     permissions:
       security-events: write
       actions: read
@@ -27,14 +117,14 @@ jobs:
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
-          virtualenvs-path: .venv
+          virtualenvs-path: ${{ env.working-directory }}/.venv
           installer-parallel: true
 
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v4
         with:
-          path: .venv
+          path: ${{ env.working-directory }}/.venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock')}}
 
       - name: Install dependencies
@@ -43,6 +133,7 @@ jobs:
         working-directory: ${{ env.working-directory }}
 
       - name: Install project
+        if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true'
         run: poetry install --no-interaction
         working-directory: ${{ env.working-directory }}
 
@@ -58,16 +149,8 @@ jobs:
           ruff check ttfd/
         working-directory: ${{ env.working-directory }}
 
-      # - name: Run tests
-      #   run: |
-      #     source .venv/bin/activate
-      #     pytest test/
-      #     coverage report
-      #   working-directory: ${{ env.working-directory }}
-
-      - name: Security
+      - name: Security linting
         run: |
           source .venv/bin/activate
           bandit -c pyproject.toml -r ttfd
         working-directory: ${{ env.working-directory }}
-

.gitignore

@@ -133,7 +133,7 @@ node_modules/
 
 # Data
 *.db
-data/
+/server/data/
 
 # Generated test files
 test/test_counts.py

docker-compose.yml

@@ -34,7 +34,7 @@ services:
     ports:
       - "8000:8000"
     command: |
-      bash -c 'poetry run alembic upgrade head && poetry run admin create -u 738 -n "James Collier" -i "https://services.vib.be/api/document/profilepic-url/53" &&  poetry run admin promote -u 738 && poetry run hypercorn --bind "0.0.0.0:8000" --reload ttfd.main:app'
+      bash -c 'poetry run alembic upgrade head && poetry run admin create -u 738 -n "James Collier" -i "https://services.vib.be/api/document/profilepic-url/53" &&  poetry run admin promote -u 738 && poetry run hypercorn ttfd.main:app --bind "0.0.0.0:8000" --log-level=debug --reload --access-logfile=-'
 
 volumes:
   pgdata: {}

server/dries/TTFD.py

@@ -91,6 +91,7 @@
 
 
 def do_the_thing(data,
+                 tracer_metabolite_id,
                  gauge_metabolite_id,
                  tracer_labeled_atoms,
                  gauge_num_label_filter,
@@ -102,7 +103,7 @@ def do_the_thing(data,
     #i_f = open("data/ALPHA-GLUCOSE__L-LACTATE__30.newformat", 'r')
     tracer_labeled_atoms = [str(tla) for tla in tracer_labeled_atoms]
     path_list = []
-    
+
     results = []
     for (idx, datum) in enumerate(data):
         reaction_info = datum.reaction_info
@@ -111,6 +112,11 @@ def do_the_thing(data,
         extended_reaction_list = reaction_info[2]
         metabolite_path = datum.metabolite_path
         metabolite_list = datum.metabolite_list
+        if tracer_metabolite_id not in metabolite_path[0]:
+            continue
+        if gauge_metabolite_id not in metabolite_path[-1]:
+            continue
+
         end_metabolite_incorporations = metabolite_path[-1][gauge_metabolite_id][1]
         branch_info = datum.branch_info
         loop_info = datum.loop_info
@@ -308,4 +314,4 @@ def do_the_thing(data,
 
 
 if __name__ == '__main__':
-    do_the_thing("L-LACTATE")
+    do_the_thing("L-LACTATE")

server/dries/counts.py

@@ -4,19 +4,29 @@
 from dries.TTFD import do_the_thing
 
 
-def counts_per_label(data: List[Datum], end_metabolite, start: List[int]):
+def counts_per_label(data: List[Datum], tracer: str, gauge: str, start: List[int]):
     counts = {}
-    for label in range(1, 4):
-        results = do_the_thing(data, end_metabolite, start, [label])
+    for label in range(1, 50):
+        results = do_the_thing(data, tracer, gauge, start, [label])
         if len(results) > 0:
             counts[label] = len(results)
 
     return counts
 
+def mklabeled(gauge: str, labeled_elements: list[int]) -> str:
+    size = {
+        "L-LACTATE": 6,
+        "GLT": 10,
+        "UMP": 21,
+        "RIBULOSE-5P": 14,
+        "PRO": 8,
+    }[gauge]
+    return "_".join("x" if p in labeled_elements else "" for p in range(size))
 
 def counts_per_label_and_pathways(
     data: List[Datum],
-    end_metabolite,
+    tracer: str,
+    gauge: str,
     start: List[int],
     label_count: int,
     labeled_elements: list[int] | None,
@@ -25,32 +35,35 @@ def counts_per_label_and_pathways(
     labeled = (
         None
         if labeled_elements is None
-        else ["_".join(["x" if p in labeled_elements else "" for p in range(6)])]
+        else [mklabeled(gauge, labeled_elements)]
     )
+
     for pathway_count in range(1, 9):
         results = do_the_thing(
             data,
-            end_metabolite,
-            start,
-            [label_count],
-            [pathway_count],
-            include_labeled_elements=labeled,
+            tracer,
+            gauge,
+            start,  # [1,2] (`tracer_labeled_atoms`)
+            [label_count], # [2] (`gauge_num_label_filter`)
+            [pathway_count], # [1] (`num_pathways_filter`)
+            include_labeled_elements=labeled, # ["x_x____"] (`include_labeled_elements`)
         )
+        # print(pathway_count, results)
         if len(results) > 0:
             counts[pathway_count] = len(results)
 
     return counts
 
 
 def counts_per_position(
-    data: List[Datum], end_metabolite, start: List[int], positions: List[int]
+    data: List[Datum], tracer: str, gauge: str, start: List[int], positions: List[int]
 ):
     labeling_mask = "_".join(["x" if p in positions else "" for p in range(6)])
     counts = {}
 
     for pathway_count in range(1, 9):
         results = do_the_thing(
-            data, end_metabolite, start, None, [pathway_count], [labeling_mask]
+            data, tracer, gauge, start, None, [pathway_count], [labeling_mask]
         )
         if len(results) > 0:
             counts[pathway_count] = len(results)
@@ -59,7 +72,7 @@ def counts_per_position(
 
 
 def counts_per_labeled_position(
-    data: List[Datum], end_metabolite, start: List[Tuple[int, int]]
+    data: List[Datum], tracer, gauge, start: List[Tuple[int, int]]
 ):
     labels, positions = zip(*start)
     labeling_mask = "_".join(
@@ -69,7 +82,7 @@ def counts_per_labeled_position(
 
     for pathway_count in range(1, 9):
         results = do_the_thing(
-            data, end_metabolite, start, None, [pathway_count], None, [labeling_mask]
+            data, tracer, gauge, start, None, [pathway_count], None, [labeling_mask]
         )
         if len(results) > 0:
             counts[pathway_count] = len(results)

server/dries/preload.py

@@ -12,7 +12,7 @@ class Datum:
     loop_info: Any
     pathway_info: Any
     expanded_reaction_paths: Any
-    metabolite_list: List[Any]
+    metabolite_list: list[Any]
 
 
 def expand_reaction_paths(compressed_reaction_paths):
@@ -22,42 +22,42 @@ def expand_reaction_paths(compressed_reaction_paths):
     return expanded_reaction_paths
 
 
-def preload(filename: str) -> List[Datum]:
+def preload(readable) -> list[Datum]:
     data = []
-    with open(filename) as f:
-        while True:
-            reaction_line = f.readline().strip()
-            if not reaction_line:
-                break
-            reaction_info = literal_eval(reaction_line)
-            compressed_reaction_path = reaction_info[0]
-
-            metabolite_line = f.readline().strip()
-            if not metabolite_line:
-                break
-            metabolite_path = literal_eval(metabolite_line)
-
-            branch_line = f.readline().strip()
-            if not branch_line:
-                break
-
-            loop_line = f.readline().strip()
-            if not loop_line:
-                break
-
-            pathways_line = f.readline().strip()
-            if not pathways_line:
-                break
-
-            data.append(
-                Datum(
-                    reaction_info,
-                    metabolite_path,
-                    literal_eval(branch_line),
-                    literal_eval(loop_line),
-                    literal_eval(pathways_line),
-                    expand_reaction_paths(compressed_reaction_path),
-                    list(chain.from_iterable([list(m) for m in metabolite_path])),
-                )
+
+    while True:
+        reaction_line = readable.readline().strip()
+        if not reaction_line:
+            break
+        reaction_info = literal_eval(reaction_line)
+        compressed_reaction_path = reaction_info[0]
+
+        metabolite_line = readable.readline().strip()
+        if not metabolite_line:
+            break
+        metabolite_path = literal_eval(metabolite_line)
+
+        branch_line = readable.readline().strip()
+        if not branch_line:
+            break
+
+        loop_line = readable.readline().strip()
+        if not loop_line:
+            break
+
+        pathways_line = readable.readline().strip()
+        if not pathways_line:
+            break
+
+        data.append(
+            Datum(
+                reaction_info,
+                metabolite_path,
+                literal_eval(branch_line),
+                literal_eval(loop_line),
+                literal_eval(pathways_line),
+                expand_reaction_paths(compressed_reaction_path),
+                list(chain.from_iterable([list(m) for m in metabolite_path])),
             )
+        )
     return data