From f672e596e44d508848b3ce80149ef549469c5137 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 12:33:28 +0200 Subject: [PATCH 01/14] rename fields for consistency and clarity --- src/routers/flows.py | 4 ++-- src/schemas/flows.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/routers/flows.py b/src/routers/flows.py index 0bb02cb7..3a9ec14a 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -74,8 +74,8 @@ async def get_flow( ) return Flow( - id_=flow.id, - uploader=flow.uploader, + id=flow.id, + uploader_id=flow.uploader, name=flow.name, class_name=flow.class_name, version=flow.version, diff --git a/src/schemas/flows.py b/src/schemas/flows.py index a6cd479c..b2b0813a 100644 --- a/src/schemas/flows.py +++ b/src/schemas/flows.py @@ -1,10 +1,12 @@ from __future__ import annotations from datetime import datetime -from typing import Any, TypedDict +from typing import Any from pydantic import BaseModel, ConfigDict, Field +from core.types import Identifier, TagString + class Parameter(BaseModel): name: str @@ -14,8 +16,8 @@ class Parameter(BaseModel): class Flow(BaseModel): - id_: int = Field(serialization_alias="id") - uploader: int | None + id: Identifier + uploader_id: Identifier | None = Field(serialization_alias="uploader") name: str = Field(max_length=1024) class_name: str | None = Field(max_length=256) version: int @@ -26,11 +28,11 @@ class Flow(BaseModel): dependencies: str | None parameter: list[Parameter] subflows: list[Subflow] - tag: list[str] + tag: list[TagString] model_config = ConfigDict(arbitrary_types_allowed=True) -class Subflow(TypedDict): - identifier: str | None +class Subflow(BaseModel): + alias: str | None = Field(alias="identifier") flow: Flow From 7aacbd12ca7dcc7c6c3112da0ba037407bc03447 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 13:52:39 +0200 Subject: [PATCH 02/14] Rename and type study for consistency --- src/routers/study.py | 4 ++-- src/schemas/study.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/routers/study.py b/src/routers/study.py index a6869296..a13bb9e1 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -169,7 +169,7 @@ async def get_study( study_data = await database.studies.get_study_data(study, expdb) return Study( _legacy=str_to_bool(study.legacy), - id_=study.id, + id=study.id, name=study.name, alias=study.alias, main_entity_type=study.type_, @@ -177,7 +177,7 @@ async def get_study( visibility=study.visibility, status=study.status, creation_date=study.creation_date, - creator=study.creator, + uploader_id=study.creator, data_ids=[row.data_id for row in study_data], task_ids=[row.task_id for row in study_data], run_ids=[row.run_id for row in study_data] if study.type_ == StudyType.RUN else [], diff --git a/src/schemas/study.py b/src/schemas/study.py index f88b060a..d86e3d8f 100644 --- a/src/schemas/study.py +++ b/src/schemas/study.py @@ -3,6 +3,7 @@ from pydantic import BaseModel, Field +from core.types import Identifier from schemas.core import Visibility @@ -19,7 +20,7 @@ class StudyStatus(StrEnum): class Study(BaseModel): legacy: bool = Field(default=False, exclude=True) - id_: int = Field(serialization_alias="id") + id: Identifier name: str alias: str | None main_entity_type: StudyType @@ -27,7 +28,7 @@ class Study(BaseModel): visibility: Visibility status: StudyStatus creation_date: datetime - creator: int + uploader_id: Identifier = Field(serialization_alias="creator") task_ids: list[int] run_ids: list[int] data_ids: list[int] From 6aeb97d5acb79ef16d09d2b73bb6df4c01b5ec9c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 13:55:53 +0200 Subject: [PATCH 03/14] Improve consistency for field names and typing --- src/database/evaluations.py | 2 +- src/routers/tasks.py | 2 +- src/schemas/datasets.py | 9 +++++---- src/schemas/tasks.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/database/evaluations.py b/src/database/evaluations.py index ce0d230a..d10bbaa8 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -32,7 +32,7 @@ async def get_estimation_procedures(connection: AsyncConnection) -> list[Estimat row = await connection.execute( text( """ - SELECT `id` as 'id_', `ttid` as 'task_type_id', `name`, `type` as 'type_', + SELECT `id`, `ttid` as 'task_type_id', `name`, `type` as 'type_', `repeats`, `folds`, `stratified_sampling`, `percentage` FROM estimation_procedure """, diff --git a/src/routers/tasks.py b/src/routers/tasks.py index ae01a301..ec978d81 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -494,7 +494,7 @@ async def get_task( name = f"Task {task_id}: {dataset.name} ({task_type.name})" return Task( - id_=task.task_id, + id=task.task_id, name=name, task_type_id=task.ttid, task_type=task_type.name, diff --git a/src/schemas/datasets.py b/src/schemas/datasets.py index be4795e6..34b6ac46 100644 --- a/src/schemas/datasets.py +++ b/src/schemas/datasets.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, Field, HttpUrl +from core.types import Identifier + class DatasetFileFormat(StrEnum): """Allowed file formats for data files.""" @@ -68,7 +70,7 @@ class Feature(BaseModel): class DatasetMetadata(BaseModel): """Metadata for a dataset.""" - id_: int = Field(json_schema_extra={"example": 1}, alias="id") + id: Identifier = Field(json_schema_extra={"example": 1}) visibility: Visibility = Field(json_schema_extra={"example": Visibility.PUBLIC}) status: DatasetStatus = Field(json_schema_extra={"example": DatasetStatus.ACTIVE}) @@ -115,7 +117,7 @@ class DatasetMetadata(BaseModel): description: str = Field( json_schema_extra={"example": "The original Annealing dataset from UCI."}, ) - description_version: int = Field(json_schema_extra={"example": 2}) + description_version: Identifier = Field(json_schema_extra={"example": 2}) tags: list[str] = Field(json_schema_extra={"example": ["study_1", "uci"]}, alias="tag") default_target_attribute: list[str] = Field(json_schema_extra={"example": "class"}) ignore_attribute: list[str] = Field(json_schema_extra={"example": "sensitive_feature"}) @@ -134,9 +136,8 @@ class DatasetMetadata(BaseModel): }, ) file_id: int = Field(json_schema_extra={"example": 1}) - format_: DatasetFileFormat = Field( + format: DatasetFileFormat = Field( json_schema_extra={"example": DatasetFileFormat.ARFF}, - alias="format", ) original_data_url: list[HttpUrl] | None = Field( json_schema_extra={"example": "https://www.openml.org/d/2"}, diff --git a/src/schemas/tasks.py b/src/schemas/tasks.py index 475f1a4c..2bc46718 100644 --- a/src/schemas/tasks.py +++ b/src/schemas/tasks.py @@ -10,7 +10,7 @@ class Task(BaseModel): """Metadata for a task.""" - id_: Identifier = Field(serialization_alias="id", json_schema_extra={"example": 59}) + id: Identifier = Field(json_schema_extra={"example": 59}) name: str = Field( json_schema_extra={"example": "Task 59: mfeat-pixel (Supervised Classification)"}, ) @@ -24,7 +24,7 @@ class Task(BaseModel): class EstimationProcedure(BaseModel): """Description of an evaluation protocol, e.g., cross-validation.""" - id_: Identifier = Field(serialization_alias="id") + id: Identifier task_type_id: Identifier name: str type_: str = Field(serialization_alias="type") From a7c4b57616cc0c2ac2f029d13f6cc59ebb56cb86 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 13:59:56 +0200 Subject: [PATCH 04/14] Rename and type variable names more consistently --- src/database/datasets.py | 28 +++++++++++++++++----------- src/database/flows.py | 4 ++-- src/database/runs.py | 4 ++-- src/database/studies.py | 4 ++-- src/database/tasks.py | 14 +++++++------- src/routers/datasets.py | 2 +- tests/routers/dataset_tag_test.py | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/database/datasets.py b/src/database/datasets.py index 1d6bf05d..d9c5d9ab 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -21,7 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection -async def get(id_: Identifier, connection: AsyncConnection) -> UntypedRow | None: +async def get(dataset_id: Identifier, connection: AsyncConnection) -> UntypedRow | None: row = await connection.execute( text( """ @@ -30,7 +30,7 @@ async def get(id_: Identifier, connection: AsyncConnection) -> UntypedRow | None WHERE did = :dataset_id """, ), - parameters={"dataset_id": id_}, + parameters={"dataset_id": dataset_id}, ) return row.one_or_none() @@ -80,7 +80,7 @@ async def delete_tag(dataset_id: Identifier, tag: TagString, connection: AsyncCo ) -async def get_tags_for(id_: Identifier, connection: AsyncConnection) -> list[str]: +async def get_tags_for(dataset_id: Identifier, connection: AsyncConnection) -> list[str]: row = await connection.execute( text( """ @@ -89,13 +89,19 @@ async def get_tags_for(id_: Identifier, connection: AsyncConnection) -> list[str WHERE id = :dataset_id """, ), - parameters={"dataset_id": id_}, + parameters={"dataset_id": dataset_id}, ) rows = row.all() return [row.tag for row in rows] -async def tag(id_: int, tag_: str, *, user_id: int, connection: AsyncConnection) -> None: +async def tag( + dataset_id: Identifier, + tag: str, + *, + user_id: Identifier, + connection: AsyncConnection, +) -> None: try: await connection.execute( text( @@ -105,9 +111,9 @@ async def tag(id_: int, tag_: str, *, user_id: int, connection: AsyncConnection) """, ), parameters={ - "dataset_id": id_, + "dataset_id": dataset_id, "user_id": user_id, - "tag": tag_, + "tag": tag, }, ) except IntegrityError as e: @@ -122,7 +128,7 @@ async def tag(id_: int, tag_: str, *, user_id: int, connection: AsyncConnection) async def get_description( - id_: Identifier, + dataset_id: Identifier, connection: AsyncConnection, ) -> UntypedRow | None: """Get the most recent description for the dataset.""" @@ -135,12 +141,12 @@ async def get_description( ORDER BY version DESC """, ), - parameters={"dataset_id": id_}, + parameters={"dataset_id": dataset_id}, ) return row.first() -async def get_status(id_: Identifier, connection: AsyncConnection) -> DatasetStatus: +async def get_status(dataset_id: Identifier, connection: AsyncConnection) -> DatasetStatus: """Get most recent status for the dataset.""" row = ( await connection.execute( @@ -153,7 +159,7 @@ async def get_status(id_: Identifier, connection: AsyncConnection) -> DatasetSta LIMIT 1 """, ), - parameters={"dataset_id": id_}, + parameters={"dataset_id": dataset_id}, ) ).first() return DatasetStatus(row.status) if row else DatasetStatus.IN_PREPARATION diff --git a/src/database/flows.py b/src/database/flows.py index d245d66d..b71e63eb 100644 --- a/src/database/flows.py +++ b/src/database/flows.py @@ -72,7 +72,7 @@ async def get_by_name( return row.one_or_none() -async def get(id_: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get(flow_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: row = await expdb.execute( text( """ @@ -81,6 +81,6 @@ async def get(id_: Identifier, expdb: AsyncConnection) -> UntypedRow | None: WHERE id = :flow_id """, ), - parameters={"flow_id": id_}, + parameters={"flow_id": flow_id}, ) return row.one_or_none() diff --git a/src/database/runs.py b/src/database/runs.py index 2a4d5470..7559f1f0 100644 --- a/src/database/runs.py +++ b/src/database/runs.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection -async def exist(id_: Identifier, expdb: AsyncConnection) -> bool: +async def exist(run_id: Identifier, expdb: AsyncConnection) -> bool: """Check if a run exists by ID.""" row = await expdb.execute( text( @@ -22,7 +22,7 @@ async def exist(id_: Identifier, expdb: AsyncConnection) -> bool: WHERE `rid` = :run_id """, ), - parameters={"run_id": id_}, + parameters={"run_id": run_id}, ) return bool(row.one_or_none()) diff --git a/src/database/studies.py b/src/database/studies.py index b9617164..237f66b6 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -14,7 +14,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection -async def get_by_id(id_: Identifier, connection: AsyncConnection) -> UntypedRow | None: +async def get_by_id(study_id: Identifier, connection: AsyncConnection) -> UntypedRow | None: row = await connection.execute( text( """ @@ -23,7 +23,7 @@ async def get_by_id(id_: Identifier, connection: AsyncConnection) -> UntypedRow WHERE id = :study_id """, ), - parameters={"study_id": id_}, + parameters={"study_id": study_id}, ) return row.one_or_none() diff --git a/src/database/tasks.py b/src/database/tasks.py index 74c330bb..2539abdb 100644 --- a/src/database/tasks.py +++ b/src/database/tasks.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession -async def get(id_: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get(task_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: row = await expdb.execute( text( """ @@ -27,7 +27,7 @@ async def get(id_: Identifier, expdb: AsyncConnection) -> UntypedRow | None: WHERE `task_id` = :task_id """, ), - parameters={"task_id": id_}, + parameters={"task_id": task_id}, ) return row.one_or_none() @@ -102,7 +102,7 @@ async def get_task_evaluation_measure(task_id: int, expdb: AsyncConnection) -> s async def get_input_for_task_type( - task_type_id: int, + task_type_id: Identifier, expdb: AsyncConnection, ) -> Sequence[UntypedRow]: rows = await expdb.execute( @@ -118,7 +118,7 @@ async def get_input_for_task_type( return rows.all() -async def get_input_for_task(id_: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_input_for_task(task_id: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: rows = await expdb.execute( text( """ @@ -127,7 +127,7 @@ async def get_input_for_task(id_: Identifier, expdb: AsyncConnection) -> Sequenc WHERE task_id = :task_id """, ), - parameters={"task_id": id_}, + parameters={"task_id": task_id}, ) return rows.all() @@ -155,13 +155,13 @@ async def get_tags(task_id: Identifier, session: AsyncSession) -> Sequence[TaskT async def tag( - id_: Identifier, + task_id: Identifier, tag_: TagString, *, user_id: Identifier, session: AsyncSession, ) -> None: - tag = TaskTag(entity_id=id_, uploader_id=user_id, tag=tag_) + tag = TaskTag(entity_id=task_id, uploader_id=user_id, tag=tag_) try: session.add(tag) await session.flush() diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 2718e7b0..b9728ddd 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -119,7 +119,7 @@ async def untag_dataset_like_php( ) -> dict[Literal["data_untag"], TagInfo]: """Remove a tag from the dataset with a response similar to the old PHP server.""" await untag_dataset(data_id, tag, user, expdb_db) - tags = await database.datasets.get_tags_for(id_=data_id, connection=expdb_db) + tags = await database.datasets.get_tags_for(dataset_id=data_id, connection=expdb_db) tag_info: TagInfo = {"id": str(data_id)} if len(tags) == 1: tag_info["tag"] = tags[0] diff --git a/tests/routers/dataset_tag_test.py b/tests/routers/dataset_tag_test.py index 6c8201fa..4e31d335 100644 --- a/tests/routers/dataset_tag_test.py +++ b/tests/routers/dataset_tag_test.py @@ -49,7 +49,7 @@ async def test_dataset_tag( result = await tag_dataset(data_id=dataset_id, tag=tag, user=user, expdb_db=expdb_test) assert result == {"data_tag": {"id": str(dataset_id), "tag": [tag]}} - tags = await get_tags_for(id_=dataset_id, connection=expdb_test) + tags = await get_tags_for(dataset_id=dataset_id, connection=expdb_test) assert tag in tags From 34e5ba39d71c78d5857f9fc76d9960f7c97762d6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 14:09:18 +0200 Subject: [PATCH 05/14] Improve consistency of naming and typing identifiers --- src/database/runs.py | 14 +++++++------- src/database/studies.py | 4 ++-- src/database/tasks.py | 4 ++-- src/database/users.py | 8 ++++---- src/routers/runs.py | 4 ++-- src/routers/study.py | 2 +- src/routers/tasks.py | 2 +- src/routers/tasktype.py | 3 ++- src/schemas/datasets.py | 2 +- src/schemas/runs.py | 22 ++++++++++++---------- src/schemas/setups.py | 10 ++++++---- src/schemas/study.py | 6 +++--- tests/conftest.py | 4 ++-- tests/routers/datasets_status_test.py | 7 ++++--- 14 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/database/runs.py b/src/database/runs.py index 7559f1f0..1e4e707b 100644 --- a/src/database/runs.py +++ b/src/database/runs.py @@ -5,7 +5,7 @@ from sqlalchemy import bindparam, text -from core.types import Identifier +from core.types import Identifier, TagString from database.models.base import UntypedRow if TYPE_CHECKING: @@ -46,7 +46,7 @@ async def get(run_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: return row.one_or_none() -async def get_tags(run_id: int, expdb: AsyncConnection) -> list[str]: +async def get_tags(run_id: Identifier, expdb: AsyncConnection) -> list[TagString]: """Fetch all tags associated with a run from the `run_tag` table. The `id` column in `run_tag` refers to the run ID @@ -64,7 +64,7 @@ async def get_tags(run_id: int, expdb: AsyncConnection) -> list[str]: return [row.tag for row in rows.all()] -async def get_input_data(run_id: int, expdb: AsyncConnection) -> list[UntypedRow]: +async def get_input_data(run_id: Identifier, expdb: AsyncConnection) -> list[UntypedRow]: """Fetch the dataset(s) used as input for a run, with name and url. Joins `input_data` with `dataset` to include the dataset name and ARFF URL. @@ -83,7 +83,7 @@ async def get_input_data(run_id: int, expdb: AsyncConnection) -> list[UntypedRow return cast("list[UntypedRow]", rows.all()) -async def get_output_files(run_id: int, expdb: AsyncConnection) -> list[UntypedRow]: +async def get_output_files(run_id: Identifier, expdb: AsyncConnection) -> list[UntypedRow]: """Fetch output files attached to a run from the `runfile` table. Typical entries include the description XML and predictions ARFF. @@ -103,10 +103,10 @@ async def get_output_files(run_id: int, expdb: AsyncConnection) -> list[UntypedR async def get_evaluations( - run_id: int, + run_id: Identifier, expdb: AsyncConnection, *, - evaluation_engine_ids: list[int], + evaluation_engine_ids: list[Identifier], ) -> list[UntypedRow]: """Fetch evaluation metric results for a run. @@ -142,7 +142,7 @@ async def get_evaluations( return cast("list[UntypedRow]", rows.all()) -async def get_trace(run_id: int, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_trace(run_id: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: """Get trace rows for a run from the trace table.""" rows = await expdb.execute( text( diff --git a/src/database/studies.py b/src/database/studies.py index 237f66b6..27120ffa 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -148,8 +148,8 @@ async def attach_run( async def attach_tasks( *, - study_id: int, - task_ids: list[int], + study_id: Identifier, + task_ids: list[Identifier], user: User, connection: AsyncConnection, ) -> None: diff --git a/src/database/tasks.py b/src/database/tasks.py index 2539abdb..1649fc63 100644 --- a/src/database/tasks.py +++ b/src/database/tasks.py @@ -58,7 +58,7 @@ async def get_task_type(task_type_id: Identifier, expdb: AsyncConnection) -> Unt return row.one_or_none() -async def get_task_type_name(task_id: int, expdb: AsyncConnection) -> str | None: +async def get_task_type_name(task_id: Identifier, expdb: AsyncConnection) -> str | None: """Fetch the human-readable task type name for the task associated with a run. Joins `task` and `task_type` on `ttid` to resolve the name @@ -79,7 +79,7 @@ async def get_task_type_name(task_id: int, expdb: AsyncConnection) -> str | None return result.name if result else None -async def get_task_evaluation_measure(task_id: int, expdb: AsyncConnection) -> str | None: +async def get_task_evaluation_measure(task_id: Identifier, expdb: AsyncConnection) -> str | None: """Fetch the evaluation measure configured for a task, if any. Queries `task_inputs` for the row where `input = 'evaluation_measures'`. diff --git a/src/database/users.py b/src/database/users.py index 3e46ab08..9924cbba 100644 --- a/src/database/users.py +++ b/src/database/users.py @@ -47,7 +47,7 @@ async def get_user( *, connection: AsyncConnection, api_key: APIKey | None = None, - user_id: int | None = None, + user_id: Identifier | None = None, ) -> User | None: """Fetch the full user by either api_key or user_id.""" if (api_key is None) == (user_id is None): @@ -153,7 +153,7 @@ async def is_admin(self) -> bool: return UserGroup.ADMIN in await self.get_groups() -async def exists_by_id(*, user_id: int, connection: AsyncConnection) -> bool: +async def exists_by_id(*, user_id: Identifier, connection: AsyncConnection) -> bool: row = await connection.execute( text("SELECT 1 FROM users WHERE id = :user_id LIMIT 1"), parameters={"user_id": user_id}, @@ -161,7 +161,7 @@ async def exists_by_id(*, user_id: int, connection: AsyncConnection) -> bool: return row.one_or_none() is not None -async def has_user_references(*, user_id: int, expdb: AsyncConnection) -> bool: +async def has_user_references(*, user_id: Identifier, expdb: AsyncConnection) -> bool: """Return ``True`` if any ``expdb`` row still references ``user_id``.""" row = await expdb.execute( text( @@ -190,7 +190,7 @@ async def has_user_references(*, user_id: int, expdb: AsyncConnection) -> bool: return bool(row.scalar_one()) -async def delete_user_rows(*, user_id: int, userdb: AsyncConnection) -> None: +async def delete_user_rows(*, user_id: Identifier, userdb: AsyncConnection) -> None: """Remove group memberships then the user row (openml user database).""" await userdb.execute( text("DELETE FROM users_groups WHERE user_id = :user_id"), diff --git a/src/routers/runs.py b/src/routers/runs.py index 8484b5e2..aaa6cc6d 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -144,7 +144,7 @@ def _normalise_value(v: object) -> object: @router.get("/{run_id}", response_model_exclude_none=True) async def get_run( - run_id: int, + run_id: Identifier, expdb: Annotated[AsyncConnection, Depends(expdb_connection)], userdb: Annotated[AsyncConnection, Depends(userdb_connection)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], @@ -169,7 +169,7 @@ async def get_run( return Run( run_id=run_id, - uploader=run.uploader, + uploader_id=run.uploader, uploader_name=ctx.uploader_name, task_id=run.task_id, task_type=ctx.task_type, diff --git a/src/routers/study.py b/src/routers/study.py index a13bb9e1..8fa967a1 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -62,7 +62,7 @@ async def _get_study_raise_otherwise( class AttachDetachResponse(BaseModel): """Response format for attaching or detaching an entity from a study.""" - study_id: int + study_id: Identifier main_entity_type: StudyType diff --git a/src/routers/tasks.py b/src/routers/tasks.py index ec978d81..679b135c 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -452,7 +452,7 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 @router.get("/{task_id}") async def get_task( - task_id: int, + task_id: Identifier, expdb: Annotated[AsyncConnection, Depends(expdb_connection)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> Task: diff --git a/src/routers/tasktype.py b/src/routers/tasktype.py index a92348a9..575c78ea 100644 --- a/src/routers/tasktype.py +++ b/src/routers/tasktype.py @@ -10,6 +10,7 @@ from fastapi import APIRouter, Depends from core.errors import TaskTypeNotFoundError +from core.types import Identifier from database.tasks import get_input_for_task_type, get_task_types from database.tasks import get_task_type as db_get_task_type from routers.dependencies import expdb_connection @@ -50,7 +51,7 @@ async def list_task_types( @router.get(path="/{task_type_id}") async def get_task_type( - task_type_id: int, + task_type_id: Identifier, expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[Literal["task_type"], dict[str, str | None | list[str] | list[dict[str, str]]]]: """Return a detailed description for the given task type, including expected inputs.""" diff --git a/src/schemas/datasets.py b/src/schemas/datasets.py index 34b6ac46..0a8fb167 100644 --- a/src/schemas/datasets.py +++ b/src/schemas/datasets.py @@ -135,7 +135,7 @@ class DatasetMetadata(BaseModel): "description": "URL of the parquet dataset data file.", }, ) - file_id: int = Field(json_schema_extra={"example": 1}) + file_id: Identifier = Field(json_schema_extra={"example": 1}) format: DatasetFileFormat = Field( json_schema_extra={"example": DatasetFileFormat.ARFF}, ) diff --git a/src/schemas/runs.py b/src/schemas/runs.py index 15d887ac..17043098 100644 --- a/src/schemas/runs.py +++ b/src/schemas/runs.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, ConfigDict, Field +from core.types import Identifier, TagString + class TraceIteration(BaseModel): """A single trace iteration for a run.""" @@ -17,7 +19,7 @@ class TraceIteration(BaseModel): class RunTrace(BaseModel): """Trace data for a run.""" - run_id: int + run_id: Identifier trace: list[TraceIteration] @@ -31,7 +33,7 @@ class ParameterSetting(BaseModel): name: str value: str | None - component: int # = input.implementation_id (flow_id of the owning component) + component: Identifier # = input.implementation_id (flow_id of the owning component) class InputDataset(BaseModel): @@ -41,7 +43,7 @@ class InputDataset(BaseModel): from the `dataset` table and match the values PHP returns. """ - did: int + did: Identifier name: str url: str # ARFF download URL stored in dataset.url @@ -57,7 +59,7 @@ class OutputFile(BaseModel): could create new datasets. It is intentionally omitted in this implementation. """ - file_id: int + file_id: Identifier name: str # label as stored in runfile.field, e.g. "description", "predictions" @@ -89,20 +91,20 @@ class Run(BaseModel): model_config = ConfigDict(populate_by_name=True) - run_id: int - uploader: int # user ID of the uploader + run_id: Identifier + uploader_id: Identifier = Field(serialization_alias="uploader") uploader_name: str | None - task_id: int + task_id: Identifier task_type: str | None # e.g. "Supervised Classification" task_evaluation_measure: str | None - flow_id: int | None = None + flow_id: Identifier | None = None flow_name: str | None - setup_id: int | None = None + setup_id: Identifier | None = None setup_string: str | None # human-readable description of the setup parameter_setting: list[ParameterSetting] # Serialized as "error" in JSON to match the PHP response key. # At the Python level we keep the name error_message for clarity. error_message: list[str] = Field(serialization_alias="error") - tag: list[str] + tag: list[TagString] input_data: list[InputDataset] output_data: OutputData diff --git a/src/schemas/setups.py b/src/schemas/setups.py index b4869dbe..5b97bb89 100644 --- a/src/schemas/setups.py +++ b/src/schemas/setups.py @@ -2,12 +2,14 @@ from pydantic import BaseModel, ConfigDict +from core.types import Identifier + class SetupParameter(BaseModel): """Schema representing an individual parameter within a setup.""" - id: int - flow_id: int + id: Identifier + flow_id: Identifier flow_name: str full_name: str parameter_name: str @@ -22,8 +24,8 @@ class SetupParameter(BaseModel): class SetupParameters(BaseModel): """Schema representing the grouped properties of a setup and its parameters.""" - setup_id: int - flow_id: int + setup_id: Identifier + flow_id: Identifier parameter: list[SetupParameter] | None = None model_config = ConfigDict(from_attributes=True) diff --git a/src/schemas/study.py b/src/schemas/study.py index d86e3d8f..8c87b756 100644 --- a/src/schemas/study.py +++ b/src/schemas/study.py @@ -55,7 +55,7 @@ class CreateStudy(BaseModel): description="Whether it is a collection of runs (study) or tasks (benchmarking suite).", examples=[StudyType.TASK], ) - benchmark_suite: int | None = Field( + benchmark_suite: Identifier | None = Field( # For study, refers to the benchmarking suite default=None, description="The benchmarking suite this study is based on, if any.", @@ -77,7 +77,7 @@ class CreateStudy(BaseModel): min_length=1, max_length=4096, ) - tasks: list[int] = Field( + tasks: list[Identifier] = Field( default_factory=list, description=( "Tasks to include in the study, can only be specified if `runs` is empty." @@ -85,7 +85,7 @@ class CreateStudy(BaseModel): ), examples=[[1, 2, 3]], ) - runs: list[int] = Field( + runs: list[Identifier] = Field( default_factory=list, description=( "Runs to include in the study, can only be specified if `tasks` is empty." diff --git a/tests/conftest.py b/tests/conftest.py index ded9c543..ea893f68 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -246,7 +246,7 @@ async def create_dataset( class Flow(NamedTuple): """To be replaced by an actual ORM class.""" - id: int + id: Identifier name: str external_version: str @@ -294,7 +294,7 @@ def temporary_tags( async def _temporary_tags( table: str, tags: Iterable[str], - identifier: int, + identifier: Identifier, *, persist: bool = False, ) -> AsyncIterator[None]: diff --git a/tests/routers/datasets_status_test.py b/tests/routers/datasets_status_test.py index 581465a2..0aee48a5 100644 --- a/tests/routers/datasets_status_test.py +++ b/tests/routers/datasets_status_test.py @@ -6,6 +6,7 @@ import pytest from core.errors import DatasetAdminOnlyError, DatasetNotOwnedError +from core.types import Identifier from routers.datasets import update_dataset_status from schemas.datasets import DatasetStatus from tests import constants @@ -28,7 +29,7 @@ async def test_update_status_via_api(py_api: httpx.AsyncClient) -> None: @pytest.mark.mut @pytest.mark.parametrize("dataset_id", [3, 4]) async def test_dataset_status_update_active_to_deactivated( - dataset_id: int, expdb_test: AsyncConnection + dataset_id: Identifier, expdb_test: AsyncConnection ) -> None: result = await update_dataset_status( dataset_id=dataset_id, @@ -83,7 +84,7 @@ async def test_dataset_status_update_deactivated_to_active( @pytest.mark.parametrize("dataset_id", [1, 33, 131]) async def test_dataset_status_non_admin_cannot_activate( - dataset_id: int, + dataset_id: Identifier, expdb_test: AsyncConnection, ) -> None: with pytest.raises(DatasetAdminOnlyError): @@ -97,7 +98,7 @@ async def test_dataset_status_non_admin_cannot_activate( @pytest.mark.parametrize("dataset_id", [1, 2]) async def test_dataset_status_non_owner_cannot_deactivate( - dataset_id: int, + dataset_id: Identifier, expdb_test: AsyncConnection, ) -> None: with pytest.raises(DatasetNotOwnedError): From 7c21f85336e1a86fb93109a79ef0945d0a488e1f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 14:17:15 +0200 Subject: [PATCH 06/14] Use `Identifier` typing for identifiers --- tests/routers/dataset_tag_test.py | 3 ++- tests/routers/datasets_features_test.py | 3 ++- tests/routers/datasets_get_test.py | 5 +++-- tests/routers/datasets_list_datasets_test.py | 3 ++- tests/routers/datasets_qualities_test.py | 4 +++- tests/routers/flows_exists_test.py | 3 ++- tests/routers/flows_get_test.py | 3 ++- tests/routers/runs_get_test.py | 11 ++++++----- tests/routers/runs_trace_test.py | 9 +++++---- tests/routers/setups_get_test.py | 3 ++- tests/routers/study_attach_test.py | 5 +++-- tests/routers/tag_test_helper.py | 3 ++- tests/routers/task_get_test.py | 3 ++- tests/routers/task_list_test.py | 3 ++- tests/routers/task_tag_test.py | 3 ++- tests/routers/task_type_get_test.py | 3 ++- tests/routers/users_delete_test.py | 3 ++- 17 files changed, 44 insertions(+), 26 deletions(-) diff --git a/tests/routers/dataset_tag_test.py b/tests/routers/dataset_tag_test.py index 4e31d335..8ff37edc 100644 --- a/tests/routers/dataset_tag_test.py +++ b/tests/routers/dataset_tag_test.py @@ -4,6 +4,7 @@ import pytest from core.errors import DatasetNotFoundError, TagAlreadyExistsError +from core.types import Identifier from database.datasets import get_tags_for from database.users import User from routers.datasets import tag_dataset @@ -117,7 +118,7 @@ async def test_dataset_tag_fails_if_dataset_does_not_exist(expdb_test: AsyncConn ids=["typically existing tag", "new tag"], ) async def test_dataset_tag_response_is_identical( - dataset_id: int, + dataset_id: Identifier, tag: str, api_key: str, py_api: httpx.AsyncClient, diff --git a/tests/routers/datasets_features_test.py b/tests/routers/datasets_features_test.py index f4e56d2c..df552604 100644 --- a/tests/routers/datasets_features_test.py +++ b/tests/routers/datasets_features_test.py @@ -8,6 +8,7 @@ import pytest from core.errors import DatasetNoAccessError, DatasetNotFoundError, DatasetProcessingError +from core.types import Identifier from database.users import User from routers.datasets import get_dataset_features from tests.users import ADMIN_USER, DATASET_130_OWNER @@ -117,7 +118,7 @@ async def test_dataset_features_dataset_does_not_exist(expdb_test: AsyncConnecti list(range(1, 130)), ) async def test_datasets_feature_is_identical( - data_id: int, + data_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, ) -> None: diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index cf38c0dc..73c600f1 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -11,6 +11,7 @@ import tests.constants from core.errors import DatasetNoAccessError, DatasetNotFoundError +from core.types import Identifier from database.users import User from routers.datasets import get_dataset from schemas.datasets import DatasetMetadata @@ -95,7 +96,7 @@ async def test_dataset_no_500_with_multiple_processing_entries( [138, 100_000], ) async def test_get_dataset_not_found( - dataset_id: int, + dataset_id: Identifier, expdb_test: AsyncConnection, user_test: AsyncConnection, ) -> None: @@ -160,7 +161,7 @@ async def test_private_dataset_access( range(1, 132), ) async def test_dataset_response_is_identical( # noqa: C901, PLR0912 - dataset_id: int, + dataset_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, ) -> None: diff --git a/tests/routers/datasets_list_datasets_test.py b/tests/routers/datasets_list_datasets_test.py index 89ec474f..0c307b2f 100644 --- a/tests/routers/datasets_list_datasets_test.py +++ b/tests/routers/datasets_list_datasets_test.py @@ -9,6 +9,7 @@ from hypothesis import strategies as st from core.errors import NoResultsError +from core.types import Identifier from database.users import User from routers.datasets import DatasetStatusFilter, list_datasets from routers.dependencies import LIMIT_DEFAULT, Pagination @@ -265,7 +266,7 @@ async def test_list_data_version_no_result(expdb_test: AsyncConnection) -> None: [(1, 59), (2, 34), (16, 1)], ) async def test_list_uploader( - user_id: int, count: int, user: User, expdb_test: AsyncConnection + user_id: Identifier, count: int, user: User, expdb_test: AsyncConnection ) -> None: # The dataset of user 16 is private, so can not be retrieved by other users. owner_user_id = 16 diff --git a/tests/routers/datasets_qualities_test.py b/tests/routers/datasets_qualities_test.py index d67e54a9..8f2bf549 100644 --- a/tests/routers/datasets_qualities_test.py +++ b/tests/routers/datasets_qualities_test.py @@ -6,6 +6,8 @@ import deepdiff import pytest +from core.types import Identifier + if TYPE_CHECKING: import httpx @@ -131,7 +133,7 @@ async def test_get_quality(py_api: httpx.AsyncClient) -> None: [*list(set(range(1, 133))), 9999999], ) async def test_get_quality_identical( - data_id: int, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient + data_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient ) -> None: py_response, php_response = await asyncio.gather( py_api.get(f"/datasets/qualities/{data_id}"), diff --git a/tests/routers/flows_exists_test.py b/tests/routers/flows_exists_test.py index c5889412..e86ea414 100644 --- a/tests/routers/flows_exists_test.py +++ b/tests/routers/flows_exists_test.py @@ -6,6 +6,7 @@ import pytest from core.errors import FlowNotFoundError +from core.types import Identifier from routers.flows import flow_exists from tests.conftest import Flow @@ -62,7 +63,7 @@ async def test_flow_exists_calls_db_correctly( [1, 2], ) async def test_flow_exists_processes_found( - flow_id: int, + flow_id: Identifier, mocker: MockerFixture, expdb_test: AsyncConnection, ) -> None: diff --git a/tests/routers/flows_get_test.py b/tests/routers/flows_get_test.py index 862bb702..799760cb 100644 --- a/tests/routers/flows_get_test.py +++ b/tests/routers/flows_get_test.py @@ -9,6 +9,7 @@ nested_remove_single_element_list, nested_str_to_num, ) +from core.types import Identifier if TYPE_CHECKING: import httpx @@ -322,7 +323,7 @@ async def test_get_flow_with_subflow(py_api: httpx.AsyncClient) -> None: range(1, 16), ) async def test_get_flow_equal( - flow_id: int, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient + flow_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient ) -> None: py_response, php_response = await asyncio.gather( py_api.get(f"/flows/{flow_id}"), diff --git a/tests/routers/runs_get_test.py b/tests/routers/runs_get_test.py index f9d897c5..13e369bc 100644 --- a/tests/routers/runs_get_test.py +++ b/tests/routers/runs_get_test.py @@ -10,6 +10,7 @@ import pytest from core.conversions import nested_num_to_str, nested_remove_single_element_list +from core.types import Identifier from routers.runs import _build_evaluations # ── Fixtures assume run 24 exists in the test DB (confirmed in research) ── @@ -135,10 +136,10 @@ async def test_get_run_non_empty_error(py_api: httpx.AsyncClient) -> None: # Since the test database does not have a run with an error, we mock the DB fetch class MockRunRow(NamedTuple): - rid: int - uploader: int - setup: int - task_id: int + rid: Identifier + uploader: Identifier + setup: Identifier + task_id: Identifier error_message: str mock_row = MockRunRow( @@ -225,7 +226,7 @@ def _normalize_py_run(py_run: dict[str, Any]) -> dict[str, Any]: @pytest.mark.parametrize("run_id", _RUN_IDS) async def test_get_run_equal( - run_id: int, + run_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, ) -> None: diff --git a/tests/routers/runs_trace_test.py b/tests/routers/runs_trace_test.py index c7366e44..7849c21e 100644 --- a/tests/routers/runs_trace_test.py +++ b/tests/routers/runs_trace_test.py @@ -9,13 +9,14 @@ from core.conversions import nested_num_to_str from core.errors import RunNotFoundError, RunTraceNotFoundError +from core.types import Identifier if TYPE_CHECKING: import httpx @pytest.mark.parametrize("run_id", [34]) -async def test_get_run_trace_success(run_id: int, py_api: httpx.AsyncClient) -> None: +async def test_get_run_trace_success(run_id: Identifier, py_api: httpx.AsyncClient) -> None: """Test that trace data is returned for a run that has trace entries.""" response = await py_api.get(f"/run/trace/{run_id}") assert response.status_code == HTTPStatus.OK @@ -32,7 +33,7 @@ async def test_get_run_trace_success(run_id: int, py_api: httpx.AsyncClient) -> @pytest.mark.parametrize("run_id", [24]) -async def test_get_run_trace_no_trace(run_id: int, py_api: httpx.AsyncClient) -> None: +async def test_get_run_trace_no_trace(run_id: Identifier, py_api: httpx.AsyncClient) -> None: """Test that 404 is returned for a run that exists but has no trace.""" response = await py_api.get(f"/run/trace/{run_id}") assert response.status_code == HTTPStatus.NOT_FOUND @@ -44,7 +45,7 @@ async def test_get_run_trace_no_trace(run_id: int, py_api: httpx.AsyncClient) -> @pytest.mark.parametrize("run_id", [999999]) -async def test_get_run_trace_run_not_found(run_id: int, py_api: httpx.AsyncClient) -> None: +async def test_get_run_trace_run_not_found(run_id: Identifier, py_api: httpx.AsyncClient) -> None: """Test that 404 is returned when the run does not exist.""" response = await py_api.get(f"/run/trace/{run_id}") assert response.status_code == HTTPStatus.NOT_FOUND @@ -60,7 +61,7 @@ async def test_get_run_trace_run_not_found(run_id: int, py_api: httpx.AsyncClien @pytest.mark.parametrize("run_id", _SERVER_RUNS) async def test_get_run_trace_equal( - run_id: int, + run_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, ) -> None: diff --git a/tests/routers/setups_get_test.py b/tests/routers/setups_get_test.py index 646fa0cc..4402c3c6 100644 --- a/tests/routers/setups_get_test.py +++ b/tests/routers/setups_get_test.py @@ -6,6 +6,7 @@ import pytest from core.conversions import nested_remove_values, nested_str_to_num +from core.types import Identifier if TYPE_CHECKING: import httpx @@ -45,7 +46,7 @@ async def test_get_setup_response_is_identical_setup_doesnt_exist( @pytest.mark.parametrize("setup_id", range(1, 125)) async def test_get_setup_response_is_identical( - setup_id: int, + setup_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, ) -> None: diff --git a/tests/routers/study_attach_test.py b/tests/routers/study_attach_test.py index dcb68704..ad675d13 100644 --- a/tests/routers/study_attach_test.py +++ b/tests/routers/study_attach_test.py @@ -5,6 +5,7 @@ from sqlalchemy import text from core.errors import StudyConflictError +from core.types import Identifier from schemas.study import StudyType from tests.users import ApiKey @@ -14,8 +15,8 @@ async def _attach_tasks_to_study( - study_id: int, - task_ids: list[int], + study_id: Identifier, + task_ids: list[Identifier], api_key: str, py_api: httpx.AsyncClient, expdb_test: AsyncConnection, diff --git a/tests/routers/tag_test_helper.py b/tests/routers/tag_test_helper.py index eb8fb976..95d4ca6a 100644 --- a/tests/routers/tag_test_helper.py +++ b/tests/routers/tag_test_helper.py @@ -5,13 +5,14 @@ import pytest from core.conversions import nested_remove_single_element_list +from core.types import Identifier if TYPE_CHECKING: import httpx async def assert_tag_response_is_identical( # noqa: PLR0913 - identifier: int, + identifier: Identifier, tag: str, api_key: str, entity: str, diff --git a/tests/routers/task_get_test.py b/tests/routers/task_get_test.py index 7225061a..a7055cf6 100644 --- a/tests/routers/task_get_test.py +++ b/tests/routers/task_get_test.py @@ -10,6 +10,7 @@ nested_remove_single_element_list, nested_remove_values, ) +from core.types import Identifier if TYPE_CHECKING: import httpx @@ -68,7 +69,7 @@ async def test_get_task(py_api: httpx.AsyncClient) -> None: range(1, 1306), ) async def test_get_task_equal( - task_id: int, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient + task_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient ) -> None: py_response, php_response = await asyncio.gather( py_api.get(f"/tasks/{task_id}"), diff --git a/tests/routers/task_list_test.py b/tests/routers/task_list_test.py index 01443e10..bf52c1d9 100644 --- a/tests/routers/task_list_test.py +++ b/tests/routers/task_list_test.py @@ -7,6 +7,7 @@ from core.conversions import nested_remove_single_element_list from core.errors import NoResultsError +from core.types import Identifier from routers.dependencies import LIMIT_MAX, Pagination from routers.tasks import TaskStatusFilter, list_tasks @@ -145,7 +146,7 @@ async def test_list_tasks_filter_tag(expdb_test: AsyncConnection) -> None: @pytest.mark.parametrize("task_id", [1, 59, [1, 2, 3]]) async def test_list_tasks_filter_task_id( - task_id: int | list[int], expdb_test: AsyncConnection + task_id: Identifier | list[Identifier], expdb_test: AsyncConnection ) -> None: """Filter by task_id returns only those tasks (regardless of status).""" ids = [task_id] if isinstance(task_id, int) else task_id diff --git a/tests/routers/task_tag_test.py b/tests/routers/task_tag_test.py index 8ad52b60..13413e71 100644 --- a/tests/routers/task_tag_test.py +++ b/tests/routers/task_tag_test.py @@ -4,6 +4,7 @@ import pytest from core.errors import TASK_NOT_FOUND_DURING_TAG, TagAlreadyExistsError, TaskNotFoundError +from core.types import Identifier from database.tasks import get_tags from database.users import User from routers.tasks import tag_task @@ -109,7 +110,7 @@ async def test_task_tag_fails_if_task_does_not_exist(expdb_session: AsyncSession ids=["typically existing tag", "new tag"], ) async def test_task_tag_response_is_identical( - task_id: int, + task_id: Identifier, tag: str, api_key: str, py_api: httpx.AsyncClient, diff --git a/tests/routers/task_type_get_test.py b/tests/routers/task_type_get_test.py index b186da18..e83da56c 100644 --- a/tests/routers/task_type_get_test.py +++ b/tests/routers/task_type_get_test.py @@ -6,6 +6,7 @@ import pytest from core.errors import TaskTypeNotFoundError +from core.types import Identifier if TYPE_CHECKING: import httpx @@ -16,7 +17,7 @@ list(range(1, 12)), ) async def test_get_task_type( - ttype_id: int, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient + ttype_id: Identifier, py_api: httpx.AsyncClient, php_api: httpx.AsyncClient ) -> None: py_response, php_response = await asyncio.gather( py_api.get(f"/tasktype/{ttype_id}"), diff --git a/tests/routers/users_delete_test.py b/tests/routers/users_delete_test.py index c872c747..9de53ddb 100644 --- a/tests/routers/users_delete_test.py +++ b/tests/routers/users_delete_test.py @@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection # noqa: TC002 used at runtime by pytest fixtures from core.errors import AccountHasResourcesError, ForbiddenError, UserNotFoundError +from core.types import Identifier from database.users import UserGroup from routers.users import delete_user_account from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey @@ -26,7 +27,7 @@ async def test_delete_user_missing_auth(py_api: httpx.AsyncClient) -> None: class DisposableUser(NamedTuple): - user_id: int + user_id: Identifier api_key: str From 853f876212d126df34fdb0662a5ca12c81728981 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 14:41:36 +0200 Subject: [PATCH 07/14] Move schemas to router subdirectory --- src/core/access.py | 2 +- src/database/datasets.py | 2 +- src/database/evaluations.py | 2 +- src/database/qualities.py | 2 +- src/database/studies.py | 2 +- src/routers/datasets.py | 10 ++++++++-- src/routers/estimation_procedure.py | 2 +- src/routers/flows.py | 2 +- src/routers/qualities.py | 2 +- src/routers/runs.py | 2 +- src/{ => routers}/schemas/__init__.py | 0 src/{ => routers}/schemas/core.py | 4 ++++ src/{ => routers}/schemas/datasets.py | 0 src/{ => routers}/schemas/flows.py | 8 ++++++++ src/{ => routers}/schemas/runs.py | 0 src/{ => routers}/schemas/setups.py | 0 src/{ => routers}/schemas/study.py | 10 +++++++++- src/{ => routers}/schemas/tasks.py | 0 src/routers/setups.py | 2 +- src/routers/study.py | 4 ++-- src/routers/tasks.py | 2 +- tests/routers/datasets_get_test.py | 2 +- tests/routers/datasets_status_test.py | 2 +- tests/routers/study_attach_test.py | 2 +- 24 files changed, 45 insertions(+), 19 deletions(-) rename src/{ => routers}/schemas/__init__.py (100%) rename src/{ => routers}/schemas/core.py (79%) rename src/{ => routers}/schemas/datasets.py (100%) rename src/{ => routers}/schemas/flows.py (75%) rename src/{ => routers}/schemas/runs.py (100%) rename src/{ => routers}/schemas/setups.py (100%) rename src/{ => routers}/schemas/study.py (91%) rename src/{ => routers}/schemas/tasks.py (100%) diff --git a/src/core/access.py b/src/core/access.py index 920e920a..ad3f6ff2 100644 --- a/src/core/access.py +++ b/src/core/access.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any from database.users import User -from schemas.datasets import Visibility +from routers.schemas.datasets import Visibility if TYPE_CHECKING: from sqlalchemy.engine import Row diff --git a/src/database/datasets.py b/src/database/datasets.py index d9c5d9ab..b3f720b0 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -15,7 +15,7 @@ ForeignKeyConstraintError, ) from database.models.base import UntypedRow -from schemas.datasets import DatasetStatus, Feature +from routers.schemas.datasets import DatasetStatus, Feature if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/evaluations.py b/src/database/evaluations.py index d10bbaa8..0ff49f25 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -5,7 +5,7 @@ from core.formatting import str_to_bool from database.models.base import UntypedRow -from schemas.tasks import EstimationProcedure +from routers.schemas.tasks import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/qualities.py b/src/database/qualities.py index bb239eb3..e285d321 100644 --- a/src/database/qualities.py +++ b/src/database/qualities.py @@ -5,7 +5,7 @@ from sqlalchemy import text from core.types import Identifier -from schemas.datasets import Quality +from routers.schemas.datasets import Quality if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/studies.py b/src/database/studies.py index 27120ffa..38991faf 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -8,7 +8,7 @@ from core.types import Identifier from database.models.base import UntypedRow from database.users import User -from schemas.study import CreateStudy, StudyType +from routers.schemas.study import CreateStudy, StudyType if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/datasets.py b/src/routers/datasets.py index b9728ddd..dafaa43c 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -56,8 +56,14 @@ fetch_user_or_raise, userdb_connection, ) -from schemas.core import TagInfo -from schemas.datasets import DatasetFileFormat, DatasetMetadata, DatasetStatus, Feature, FeatureType +from routers.schemas.core import TagInfo +from routers.schemas.datasets import ( + DatasetFileFormat, + DatasetMetadata, + DatasetStatus, + Feature, + FeatureType, +) if TYPE_CHECKING: from sqlalchemy.engine import Row diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index d7e76d2f..33c942e9 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -6,7 +6,7 @@ import database.evaluations from routers.dependencies import expdb_connection -from schemas.tasks import EstimationProcedure +from routers.schemas.tasks import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/flows.py b/src/routers/flows.py index 3a9ec14a..4380efbc 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -10,7 +10,7 @@ from core.errors import FlowNotFoundError from core.types import Identifier from routers.dependencies import expdb_connection -from schemas.flows import Flow, Parameter, Subflow +from routers.schemas.flows import Flow, Parameter, Subflow if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/qualities.py b/src/routers/qualities.py index 58d08e69..9b21d23f 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -20,7 +20,7 @@ from core.types import Identifier from database.users import User from routers.dependencies import expdb_connection, fetch_user -from schemas.datasets import Quality +from routers.schemas.datasets import Quality if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/runs.py b/src/routers/runs.py index aaa6cc6d..4613df26 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -17,7 +17,7 @@ from database.models.base import UntypedRow from database.models.setups import Setup from routers.dependencies import expdb_connection, expdb_session, userdb_connection -from schemas.runs import ( +from routers.schemas.runs import ( EvaluationScore, InputDataset, OutputData, diff --git a/src/schemas/__init__.py b/src/routers/schemas/__init__.py similarity index 100% rename from src/schemas/__init__.py rename to src/routers/schemas/__init__.py diff --git a/src/schemas/core.py b/src/routers/schemas/core.py similarity index 79% rename from src/schemas/core.py rename to src/routers/schemas/core.py index b9bcb5e6..b2845d89 100644 --- a/src/schemas/core.py +++ b/src/routers/schemas/core.py @@ -1,3 +1,5 @@ +"""Schema components used in many routers.""" + from enum import StrEnum, auto from typing import NotRequired, TypedDict @@ -5,6 +7,8 @@ class Visibility(StrEnum): + """Visibility of an asset.""" + PUBLIC = auto() PRIVATE = auto() diff --git a/src/schemas/datasets.py b/src/routers/schemas/datasets.py similarity index 100% rename from src/schemas/datasets.py rename to src/routers/schemas/datasets.py diff --git a/src/schemas/flows.py b/src/routers/schemas/flows.py similarity index 75% rename from src/schemas/flows.py rename to src/routers/schemas/flows.py index b2b0813a..5affd817 100644 --- a/src/schemas/flows.py +++ b/src/routers/schemas/flows.py @@ -1,3 +1,5 @@ +"""Defines schemas for API responses relating to Flows.""" + from __future__ import annotations from datetime import datetime @@ -9,6 +11,8 @@ class Parameter(BaseModel): + """Metadata for a Hyperparameter (e.g., depth of a decision tree).""" + name: str default_value: Any data_type: str @@ -16,6 +20,8 @@ class Parameter(BaseModel): class Flow(BaseModel): + """Metadata for a Flow: anything that can "solve" a task (e.g., script or algorithm).""" + id: Identifier uploader_id: Identifier | None = Field(serialization_alias="uploader") name: str = Field(max_length=1024) @@ -34,5 +40,7 @@ class Flow(BaseModel): class Subflow(BaseModel): + """Helper component to denote the alias of subflow components.""" + alias: str | None = Field(alias="identifier") flow: Flow diff --git a/src/schemas/runs.py b/src/routers/schemas/runs.py similarity index 100% rename from src/schemas/runs.py rename to src/routers/schemas/runs.py diff --git a/src/schemas/setups.py b/src/routers/schemas/setups.py similarity index 100% rename from src/schemas/setups.py rename to src/routers/schemas/setups.py diff --git a/src/schemas/study.py b/src/routers/schemas/study.py similarity index 91% rename from src/schemas/study.py rename to src/routers/schemas/study.py index 8c87b756..ac388036 100644 --- a/src/schemas/study.py +++ b/src/routers/schemas/study.py @@ -1,24 +1,32 @@ +"""Defines schemas for API responses relating to Studies.""" + from datetime import datetime from enum import StrEnum, auto from pydantic import BaseModel, Field from core.types import Identifier -from schemas.core import Visibility +from routers.schemas.core import Visibility class StudyType(StrEnum): + """Studies currently are defined by either tasks or runs.""" + RUN = auto() TASK = auto() class StudyStatus(StrEnum): + """Status of a study.""" + ACTIVE = auto() DEACTIVATED = auto() IN_PREPARATION = auto() class Study(BaseModel): + """A Study is a collection of items. Currently restricted to tasks and runs only.""" + legacy: bool = Field(default=False, exclude=True) id: Identifier name: str diff --git a/src/schemas/tasks.py b/src/routers/schemas/tasks.py similarity index 100% rename from src/schemas/tasks.py rename to src/routers/schemas/tasks.py diff --git a/src/routers/setups.py b/src/routers/setups.py index c91f3f62..aadd7224 100644 --- a/src/routers/setups.py +++ b/src/routers/setups.py @@ -16,7 +16,7 @@ from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError from database.users import User from routers.dependencies import expdb_connection, expdb_session, fetch_user_or_raise -from schemas.setups import SetupParameters, SetupResponse +from routers.schemas.setups import SetupParameters, SetupResponse if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession diff --git a/src/routers/study.py b/src/routers/study.py index 8fa967a1..5f0f5d1f 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -22,8 +22,8 @@ from database.models.base import UntypedRow from database.users import User from routers.dependencies import expdb_connection, fetch_user, fetch_user_or_raise -from schemas.core import Visibility -from schemas.study import CreateStudy, Study, StudyStatus, StudyType +from routers.schemas.core import Visibility +from routers.schemas.study import CreateStudy, Study, StudyStatus, StudyType if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 679b135c..7c07b116 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -26,7 +26,7 @@ from database.models.base import UntypedRow from database.users import User from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise -from schemas.tasks import Task +from routers.schemas.tasks import Task if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index 73c600f1..46e1aa9a 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -14,7 +14,7 @@ from core.types import Identifier from database.users import User from routers.datasets import get_dataset -from schemas.datasets import DatasetMetadata +from routers.schemas.datasets import DatasetMetadata from tests.users import ADMIN_USER, DATASET_130_OWNER, NO_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/datasets_status_test.py b/tests/routers/datasets_status_test.py index 0aee48a5..4e9a3164 100644 --- a/tests/routers/datasets_status_test.py +++ b/tests/routers/datasets_status_test.py @@ -8,7 +8,7 @@ from core.errors import DatasetAdminOnlyError, DatasetNotOwnedError from core.types import Identifier from routers.datasets import update_dataset_status -from schemas.datasets import DatasetStatus +from routers.schemas.datasets import DatasetStatus from tests import constants from tests.users import ADMIN_USER, SOME_USER diff --git a/tests/routers/study_attach_test.py b/tests/routers/study_attach_test.py index ad675d13..928c76ab 100644 --- a/tests/routers/study_attach_test.py +++ b/tests/routers/study_attach_test.py @@ -6,7 +6,7 @@ from core.errors import StudyConflictError from core.types import Identifier -from schemas.study import StudyType +from routers.schemas.study import StudyType from tests.users import ApiKey if TYPE_CHECKING: From 910fd5e65059172b7c61a6e5ddd5e3b8e8675ce6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 15:48:12 +0200 Subject: [PATCH 08/14] partial progress to only use sesssions --- src/database/datasets.py | 86 ++++++++++---------- src/database/evaluations.py | 12 +-- src/database/flows.py | 22 ++--- src/database/qualities.py | 16 ++-- src/database/runs.py | 30 +++---- src/database/setups.py | 8 +- src/database/studies.py | 36 ++++---- src/database/tasks.py | 32 ++++---- src/database/users.py | 42 +++++----- src/routers/datasets.py | 42 +++++----- src/routers/dependencies.py | 20 +++-- src/routers/estimation_procedure.py | 6 +- src/routers/evaluations.py | 8 +- src/routers/flows.py | 8 +- src/routers/qualities.py | 10 +-- src/routers/runs.py | 22 +++-- src/routers/setups.py | 7 +- src/routers/study.py | 16 ++-- src/routers/tasks.py | 33 ++++---- src/routers/tasktype.py | 8 +- src/routers/users.py | 10 +-- tests/conftest.py | 74 ++++++++++------- tests/database/flows_test.py | 10 +-- tests/database/runs_test.py | 54 ++++++------ tests/dependencies/fetch_user_test.py | 10 +-- tests/routers/dataset_tag_test.py | 24 +++--- tests/routers/dataset_untag_test.py | 44 +++++----- tests/routers/datasets_features_test.py | 22 ++--- tests/routers/datasets_get_test.py | 30 +++---- tests/routers/datasets_list_datasets_test.py | 50 ++++++------ tests/routers/datasets_status_test.py | 26 +++--- tests/routers/flows_exists_test.py | 14 ++-- tests/routers/qualities_list_test.py | 16 ++-- tests/routers/setups_tag_test.py | 14 ++-- tests/routers/setups_untag_test.py | 10 +-- tests/routers/study_attach_test.py | 30 +++---- tests/routers/task_list_test.py | 54 ++++++------ tests/routers/users_delete_test.py | 66 +++++++-------- 38 files changed, 516 insertions(+), 506 deletions(-) diff --git a/src/database/datasets.py b/src/database/datasets.py index b3f720b0..bca13dc4 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -18,11 +18,11 @@ from routers.schemas.datasets import DatasetStatus, Feature if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def get(dataset_id: Identifier, connection: AsyncConnection) -> UntypedRow | None: - row = await connection.execute( +async def get(dataset_id: Identifier, session: AsyncSession) -> UntypedRow | None: + row = await session.execute( text( """ SELECT * @@ -30,13 +30,13 @@ async def get(dataset_id: Identifier, connection: AsyncConnection) -> UntypedRow WHERE did = :dataset_id """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) return row.one_or_none() -async def get_file(*, file_id: Identifier, connection: AsyncConnection) -> UntypedRow | None: - row = await connection.execute( +async def get_file(*, file_id: Identifier, session: AsyncSession) -> UntypedRow | None: + row = await session.execute( text( """ SELECT * @@ -44,7 +44,7 @@ async def get_file(*, file_id: Identifier, connection: AsyncConnection) -> Untyp WHERE id = :file_id """, ), - parameters={"file_id": file_id}, + params={"file_id": file_id}, ) return row.one_or_none() @@ -52,10 +52,10 @@ async def get_file(*, file_id: Identifier, connection: AsyncConnection) -> Untyp async def get_tag( dataset_id: Identifier, tag: TagString, - connection: AsyncConnection, + session: AsyncSession, ) -> UntypedRow | None: return ( - await connection.execute( + await session.execute( text( """ SELECT * @@ -63,25 +63,25 @@ async def get_tag( WHERE id = :dataset_id AND tag = :tag """, ), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) ).first() -async def delete_tag(dataset_id: Identifier, tag: TagString, connection: AsyncConnection) -> None: - await connection.execute( +async def delete_tag(dataset_id: Identifier, tag: TagString, session: AsyncSession) -> None: + await session.execute( text( """ DELETE FROM dataset_tag WHERE id = :dataset_id AND tag = :tag """, ), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) -async def get_tags_for(dataset_id: Identifier, connection: AsyncConnection) -> list[str]: - row = await connection.execute( +async def get_tags_for(dataset_id: Identifier, session: AsyncSession) -> list[str]: + row = await session.execute( text( """ SELECT * @@ -89,7 +89,7 @@ async def get_tags_for(dataset_id: Identifier, connection: AsyncConnection) -> l WHERE id = :dataset_id """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) rows = row.all() return [row.tag for row in rows] @@ -100,17 +100,17 @@ async def tag( tag: str, *, user_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> None: try: - await connection.execute( + await session.execute( text( """ INSERT INTO dataset_tag(`id`, `tag`, `uploader`) VALUES (:dataset_id, :tag, :user_id) """, ), - parameters={ + params={ "dataset_id": dataset_id, "user_id": user_id, "tag": tag, @@ -129,10 +129,10 @@ async def tag( async def get_description( dataset_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> UntypedRow | None: """Get the most recent description for the dataset.""" - row = await connection.execute( + row = await session.execute( text( """ SELECT * @@ -141,15 +141,15 @@ async def get_description( ORDER BY version DESC """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) return row.first() -async def get_status(dataset_id: Identifier, connection: AsyncConnection) -> DatasetStatus: +async def get_status(dataset_id: Identifier, session: AsyncSession) -> DatasetStatus: """Get most recent status for the dataset.""" row = ( - await connection.execute( + await session.execute( text( """ SELECT status @@ -159,7 +159,7 @@ async def get_status(dataset_id: Identifier, connection: AsyncConnection) -> Dat LIMIT 1 """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) ).first() return DatasetStatus(row.status) if row else DatasetStatus.IN_PREPARATION @@ -167,9 +167,9 @@ async def get_status(dataset_id: Identifier, connection: AsyncConnection) -> Dat async def get_latest_processing_update( dataset_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> UntypedRow | None: - row = await connection.execute( + row = await session.execute( text( """ SELECT * @@ -178,13 +178,13 @@ async def get_latest_processing_update( ORDER BY processing_date DESC """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) return row.first() -async def get_features(dataset_id: Identifier, connection: AsyncConnection) -> list[Feature]: - row = await connection.execute( +async def get_features(dataset_id: Identifier, session: AsyncSession) -> list[Feature]: + row = await session.execute( text( """ SELECT `index`,`name`,`data_type`,`is_target`, @@ -193,7 +193,7 @@ async def get_features(dataset_id: Identifier, connection: AsyncConnection) -> l WHERE `did` = :dataset_id """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) rows = row.mappings().all() return [Feature(**row, nominal_values=None) for row in rows] @@ -201,9 +201,9 @@ async def get_features(dataset_id: Identifier, connection: AsyncConnection) -> l async def get_feature_ontologies( dataset_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> dict[int, list[str]]: - rows = await connection.execute( + rows = await session.execute( text( """ SELECT `index`, `value` @@ -211,7 +211,7 @@ async def get_feature_ontologies( WHERE `did` = :dataset_id AND `description_type` = 'ontology' """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) ontologies: dict[int, list[str]] = defaultdict(list) for row in rows.mappings(): @@ -223,9 +223,9 @@ async def get_feature_values( dataset_id: Identifier, *, feature_index: int, - connection: AsyncConnection, + session: AsyncSession, ) -> list[str]: - row = await connection.execute( + row = await session.execute( text( """ SELECT `value` @@ -233,7 +233,7 @@ async def get_feature_values( WHERE `did` = :dataset_id AND `index` = :feature_index """, ), - parameters={"dataset_id": dataset_id, "feature_index": feature_index}, + params={"dataset_id": dataset_id, "feature_index": feature_index}, ) rows = row.all() return [row.value for row in rows] @@ -244,16 +244,16 @@ async def update_status( status: Literal[DatasetStatus.ACTIVE, DatasetStatus.DEACTIVATED], *, user_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> None: - await connection.execute( + await session.execute( text( """ INSERT INTO dataset_status(`did`,`status`,`status_date`,`user_id`) VALUES (:dataset, :status, :date, :user) """, ), - parameters={ + params={ "dataset": dataset_id, "status": status, "date": datetime.datetime.now(datetime.UTC), @@ -262,13 +262,13 @@ async def update_status( ) -async def remove_deactivated_status(dataset_id: Identifier, connection: AsyncConnection) -> None: - await connection.execute( +async def remove_deactivated_status(dataset_id: Identifier, session: AsyncSession) -> None: + await session.execute( text( """ DELETE FROM dataset_status WHERE `did` = :data AND `status`='deactivated' """, ), - parameters={"data": dataset_id}, + params={"data": dataset_id}, ) diff --git a/src/database/evaluations.py b/src/database/evaluations.py index 0ff49f25..f3cf93bd 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -8,14 +8,14 @@ from routers.schemas.tasks import EstimationProcedure if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def get_math_functions( function_type: str, - connection: AsyncConnection, + session: AsyncSession, ) -> Sequence[UntypedRow]: - rows = await connection.execute( + rows = await session.execute( text( """ SELECT * @@ -23,13 +23,13 @@ async def get_math_functions( WHERE `functionType` = :function_type """, ), - parameters={"function_type": function_type}, + params={"function_type": function_type}, ) return rows.all() -async def get_estimation_procedures(connection: AsyncConnection) -> list[EstimationProcedure]: - row = await connection.execute( +async def get_estimation_procedures(session: AsyncSession) -> list[EstimationProcedure]: + row = await session.execute( text( """ SELECT `id`, `ttid` as 'task_type_id', `name`, `type` as 'type_', diff --git a/src/database/flows.py b/src/database/flows.py index b71e63eb..bb05b170 100644 --- a/src/database/flows.py +++ b/src/database/flows.py @@ -7,10 +7,10 @@ from database.models.base import UntypedRow if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def get_subflows(for_flow: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_subflows(for_flow: Identifier, expdb: AsyncSession) -> Sequence[UntypedRow]: rows = await expdb.execute( text( """ @@ -19,12 +19,12 @@ async def get_subflows(for_flow: Identifier, expdb: AsyncConnection) -> Sequence WHERE parent = :flow_id """, ), - parameters={"flow_id": for_flow}, + params={"flow_id": for_flow}, ) return rows.all() -async def get_tags(flow_id: Identifier, expdb: AsyncConnection) -> list[str]: +async def get_tags(flow_id: Identifier, expdb: AsyncSession) -> list[str]: rows = await expdb.execute( text( """ @@ -33,13 +33,13 @@ async def get_tags(flow_id: Identifier, expdb: AsyncConnection) -> list[str]: WHERE id = :flow_id """, ), - parameters={"flow_id": flow_id}, + params={"flow_id": flow_id}, ) tag_rows = rows.all() return [tag.tag for tag in tag_rows] -async def get_parameters(flow_id: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_parameters(flow_id: Identifier, expdb: AsyncSession) -> Sequence[UntypedRow]: rows = await expdb.execute( text( """ @@ -48,7 +48,7 @@ async def get_parameters(flow_id: Identifier, expdb: AsyncConnection) -> Sequenc WHERE implementation_id = :flow_id """, ), - parameters={"flow_id": flow_id}, + params={"flow_id": flow_id}, ) return rows.all() @@ -56,7 +56,7 @@ async def get_parameters(flow_id: Identifier, expdb: AsyncConnection) -> Sequenc async def get_by_name( name: str, external_version: str, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> UntypedRow | None: """Get flow by name and external version.""" row = await expdb.execute( @@ -67,12 +67,12 @@ async def get_by_name( WHERE name = :name AND external_version = :external_version """, ), - parameters={"name": name, "external_version": external_version}, + params={"name": name, "external_version": external_version}, ) return row.one_or_none() -async def get(flow_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get(flow_id: Identifier, expdb: AsyncSession) -> UntypedRow | None: row = await expdb.execute( text( """ @@ -81,6 +81,6 @@ async def get(flow_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: WHERE id = :flow_id """, ), - parameters={"flow_id": flow_id}, + params={"flow_id": flow_id}, ) return row.one_or_none() diff --git a/src/database/qualities.py b/src/database/qualities.py index e285d321..37302145 100644 --- a/src/database/qualities.py +++ b/src/database/qualities.py @@ -8,11 +8,11 @@ from routers.schemas.datasets import Quality if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def get_for_dataset(dataset_id: Identifier, connection: AsyncConnection) -> list[Quality]: - row = await connection.execute( +async def get_for_dataset(dataset_id: Identifier, session: AsyncSession) -> list[Quality]: + row = await session.execute( text( """ SELECT `quality`,`value` @@ -20,7 +20,7 @@ async def get_for_dataset(dataset_id: Identifier, connection: AsyncConnection) - WHERE `data`=:dataset_id """, ), - parameters={"dataset_id": dataset_id}, + params={"dataset_id": dataset_id}, ) rows = row.all() return [Quality(name=row.quality, value=row.value) for row in rows] @@ -29,7 +29,7 @@ async def get_for_dataset(dataset_id: Identifier, connection: AsyncConnection) - async def get_for_datasets( dataset_ids: Iterable[Identifier], quality_names: Iterable[str], - connection: AsyncConnection, + session: AsyncSession, ) -> dict[int, list[Quality]]: """Don't call with user-provided input, as query is not parameterized.""" qualities_filter = ",".join(f"'{q}'" for q in quality_names) @@ -41,7 +41,7 @@ async def get_for_datasets( WHERE `data` in ({dids}) AND `quality` IN ({qualities_filter}) """, # noqa: S608 - dids and qualities are not user-provided ) - row = await connection.execute(qualities_query) + row = await session.execute(qualities_query) rows = row.all() qualities_by_id = defaultdict(list) for did, quality, value in rows: @@ -50,10 +50,10 @@ async def get_for_datasets( return dict(qualities_by_id) -async def list_all_qualities(connection: AsyncConnection) -> list[str]: +async def list_all_qualities(session: AsyncSession) -> list[str]: # The current implementation only fetches *used* qualities, otherwise you should # query: SELECT `name` FROM `quality` WHERE `type`='DataQuality' - rows = await connection.execute( + rows = await session.execute( text( """ SELECT DISTINCT(`quality`) diff --git a/src/database/runs.py b/src/database/runs.py index 1e4e707b..d47d9270 100644 --- a/src/database/runs.py +++ b/src/database/runs.py @@ -9,10 +9,10 @@ from database.models.base import UntypedRow if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def exist(run_id: Identifier, expdb: AsyncConnection) -> bool: +async def exist(run_id: Identifier, expdb: AsyncSession) -> bool: """Check if a run exists by ID.""" row = await expdb.execute( text( @@ -22,12 +22,12 @@ async def exist(run_id: Identifier, expdb: AsyncConnection) -> bool: WHERE `rid` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return bool(row.one_or_none()) -async def get(run_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get(run_id: Identifier, expdb: AsyncSession) -> UntypedRow | None: """Fetch the core run row from the `run` table. Returns the row if found, or None if no run with `run_id` exists. @@ -41,12 +41,12 @@ async def get(run_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: WHERE `rid` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return row.one_or_none() -async def get_tags(run_id: Identifier, expdb: AsyncConnection) -> list[TagString]: +async def get_tags(run_id: Identifier, expdb: AsyncSession) -> list[TagString]: """Fetch all tags associated with a run from the `run_tag` table. The `id` column in `run_tag` refers to the run ID @@ -59,12 +59,12 @@ async def get_tags(run_id: Identifier, expdb: AsyncConnection) -> list[TagString WHERE `id` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return [row.tag for row in rows.all()] -async def get_input_data(run_id: Identifier, expdb: AsyncConnection) -> list[UntypedRow]: +async def get_input_data(run_id: Identifier, expdb: AsyncSession) -> list[UntypedRow]: """Fetch the dataset(s) used as input for a run, with name and url. Joins `input_data` with `dataset` to include the dataset name and ARFF URL. @@ -78,12 +78,12 @@ async def get_input_data(run_id: Identifier, expdb: AsyncConnection) -> list[Unt WHERE `id`.`run` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return cast("list[UntypedRow]", rows.all()) -async def get_output_files(run_id: Identifier, expdb: AsyncConnection) -> list[UntypedRow]: +async def get_output_files(run_id: Identifier, expdb: AsyncSession) -> list[UntypedRow]: """Fetch output files attached to a run from the `runfile` table. Typical entries include the description XML and predictions ARFF. @@ -97,14 +97,14 @@ async def get_output_files(run_id: Identifier, expdb: AsyncConnection) -> list[U WHERE `source` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return cast("list[UntypedRow]", rows.all()) async def get_evaluations( run_id: Identifier, - expdb: AsyncConnection, + expdb: AsyncSession, *, evaluation_engine_ids: list[Identifier], ) -> list[UntypedRow]: @@ -137,12 +137,12 @@ async def get_evaluations( ).bindparams(bindparam("engine_ids", expanding=True)) rows = await expdb.execute( query, - parameters={"run_id": run_id, "engine_ids": evaluation_engine_ids}, + params={"run_id": run_id, "engine_ids": evaluation_engine_ids}, ) return cast("list[UntypedRow]", rows.all()) -async def get_trace(run_id: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_trace(run_id: Identifier, expdb: AsyncSession) -> Sequence[UntypedRow]: """Get trace rows for a run from the trace table.""" rows = await expdb.execute( text( @@ -152,6 +152,6 @@ async def get_trace(run_id: Identifier, expdb: AsyncConnection) -> Sequence[Unty WHERE `run_id` = :run_id """, ), - parameters={"run_id": run_id}, + params={"run_id": run_id}, ) return rows.all() diff --git a/src/database/setups.py b/src/database/setups.py index 2808cd98..cdb0c830 100644 --- a/src/database/setups.py +++ b/src/database/setups.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from sqlalchemy.engine import RowMapping - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def get(setup_id: Identifier, session: AsyncSession) -> Setup | None: @@ -26,9 +26,9 @@ async def get(setup_id: Identifier, session: AsyncSession) -> Setup | None: return await session.get(Setup, setup_id) -async def get_parameters(setup_id: Identifier, connection: AsyncConnection) -> list[RowMapping]: +async def get_parameters(setup_id: Identifier, session: AsyncSession) -> list[RowMapping]: """Get all parameters for setup with `setup_id` from the database.""" - rows = await connection.execute( + rows = await session.execute( text( """ SELECT @@ -48,7 +48,7 @@ async def get_parameters(setup_id: Identifier, connection: AsyncConnection) -> l ORDER BY t_impl.id, t_input.id """, ), - parameters={"setup_id": setup_id}, + params={"setup_id": setup_id}, ) return list(rows.mappings().all()) diff --git a/src/database/studies.py b/src/database/studies.py index 38991faf..d1afdac7 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -11,10 +11,10 @@ from routers.schemas.study import CreateStudy, StudyType if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def get_by_id(study_id: Identifier, connection: AsyncConnection) -> UntypedRow | None: +async def get_by_id(study_id: Identifier, connection: AsyncSession) -> UntypedRow | None: row = await connection.execute( text( """ @@ -23,12 +23,12 @@ async def get_by_id(study_id: Identifier, connection: AsyncConnection) -> Untype WHERE id = :study_id """, ), - parameters={"study_id": study_id}, + params={"study_id": study_id}, ) return row.one_or_none() -async def get_by_alias(alias: str, connection: AsyncConnection) -> UntypedRow | None: +async def get_by_alias(alias: str, connection: AsyncSession) -> UntypedRow | None: row = await connection.execute( text( """ @@ -37,12 +37,12 @@ async def get_by_alias(alias: str, connection: AsyncConnection) -> UntypedRow | WHERE alias = :study_id """, ), - parameters={"study_id": alias}, + params={"study_id": alias}, ) return row.one_or_none() -async def get_study_data(study: UntypedRow, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_study_data(study: UntypedRow, expdb: AsyncSession) -> Sequence[UntypedRow]: """Return data related to the study, content depends on the study type. For task studies: (task id, dataset id) @@ -57,7 +57,7 @@ async def get_study_data(study: UntypedRow, expdb: AsyncConnection) -> Sequence[ WHERE ts.study_id = :study_id AND ti.input = 'source_data' """, ), - parameters={"study_id": study.id}, + params={"study_id": study.id}, ) return rows.all() @@ -77,12 +77,12 @@ async def get_study_data(study: UntypedRow, expdb: AsyncConnection) -> Sequence[ WHERE rs.study_id = :study_id AND ti.input = 'source_data' """, ), - parameters={"study_id": study.id}, + params={"study_id": study.id}, ) return rows.all() -async def create(study: CreateStudy, user: User, expdb: AsyncConnection) -> int: +async def create(study: CreateStudy, user: User, expdb: AsyncSession) -> int: await expdb.execute( text( """ @@ -96,7 +96,7 @@ async def create(study: CreateStudy, user: User, expdb: AsyncConnection) -> int: ) """, ), - parameters={ + params={ "name": study.name, "alias": study.alias, "main_entity_type": study.main_entity_type, @@ -115,7 +115,7 @@ async def attach_task( task_id: Identifier, study_id: Identifier, user: User, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> None: await expdb.execute( text( @@ -124,7 +124,7 @@ async def attach_task( VALUES (:study_id, :task_id, :user_id) """, ), - parameters={"study_id": study_id, "task_id": task_id, "user_id": user.user_id}, + params={"study_id": study_id, "task_id": task_id, "user_id": user.user_id}, ) @@ -133,7 +133,7 @@ async def attach_run( run_id: Identifier, study_id: Identifier, user: User, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> None: await expdb.execute( text( @@ -142,7 +142,7 @@ async def attach_run( VALUES (:study_id, :run_id, :user_id) """, ), - parameters={"study_id": study_id, "run_id": run_id, "user_id": user.user_id}, + params={"study_id": study_id, "run_id": run_id, "user_id": user.user_id}, ) @@ -151,18 +151,18 @@ async def attach_tasks( study_id: Identifier, task_ids: list[Identifier], user: User, - connection: AsyncConnection, + session: AsyncSession, ) -> None: to_link = [(study_id, task_id, user.user_id) for task_id in task_ids] try: - await connection.execute( + await session.execute( text( """ INSERT INTO task_study (study_id, task_id, uploader) VALUES (:study_id, :task_id, :user_id) """, ), - parameters=[{"study_id": s, "task_id": t, "user_id": u} for s, t, u in to_link], + params=[{"study_id": s, "task_id": t, "user_id": u} for s, t, u in to_link], ) except Exception as e: (msg,) = e.args @@ -182,6 +182,6 @@ async def attach_runs( study_id: Identifier, run_ids: list[Identifier], user: User, - connection: AsyncConnection, + session: AsyncSession, ) -> None: raise NotImplementedError diff --git a/src/database/tasks.py b/src/database/tasks.py index 1649fc63..fb2c4533 100644 --- a/src/database/tasks.py +++ b/src/database/tasks.py @@ -15,10 +15,10 @@ from database.models.tags import TaskTag if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession -async def get(task_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get(task_id: Identifier, expdb: AsyncSession) -> UntypedRow | None: row = await expdb.execute( text( """ @@ -27,12 +27,12 @@ async def get(task_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: WHERE `task_id` = :task_id """, ), - parameters={"task_id": task_id}, + params={"task_id": task_id}, ) return row.one_or_none() -async def get_task_types(expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_task_types(expdb: AsyncSession) -> Sequence[UntypedRow]: rows = await expdb.execute( text( """ @@ -44,7 +44,7 @@ async def get_task_types(expdb: AsyncConnection) -> Sequence[UntypedRow]: return rows.all() -async def get_task_type(task_type_id: Identifier, expdb: AsyncConnection) -> UntypedRow | None: +async def get_task_type(task_type_id: Identifier, expdb: AsyncSession) -> UntypedRow | None: row = await expdb.execute( text( """ @@ -53,12 +53,12 @@ async def get_task_type(task_type_id: Identifier, expdb: AsyncConnection) -> Unt WHERE `ttid`=:ttid """, ), - parameters={"ttid": task_type_id}, + params={"ttid": task_type_id}, ) return row.one_or_none() -async def get_task_type_name(task_id: Identifier, expdb: AsyncConnection) -> str | None: +async def get_task_type_name(task_id: Identifier, expdb: AsyncSession) -> str | None: """Fetch the human-readable task type name for the task associated with a run. Joins `task` and `task_type` on `ttid` to resolve the name @@ -73,13 +73,13 @@ async def get_task_type_name(task_id: Identifier, expdb: AsyncConnection) -> str WHERE `t`.`task_id` = :task_id """, ), - parameters={"task_id": task_id}, + params={"task_id": task_id}, ) result = row.one_or_none() return result.name if result else None -async def get_task_evaluation_measure(task_id: Identifier, expdb: AsyncConnection) -> str | None: +async def get_task_evaluation_measure(task_id: Identifier, expdb: AsyncSession) -> str | None: """Fetch the evaluation measure configured for a task, if any. Queries `task_inputs` for the row where `input = 'evaluation_measures'`. @@ -95,7 +95,7 @@ async def get_task_evaluation_measure(task_id: Identifier, expdb: AsyncConnectio AND `input` = 'evaluation_measures' """, ), - parameters={"task_id": task_id}, + params={"task_id": task_id}, ) result = row.one_or_none() return result.value if result else None @@ -103,7 +103,7 @@ async def get_task_evaluation_measure(task_id: Identifier, expdb: AsyncConnectio async def get_input_for_task_type( task_type_id: Identifier, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> Sequence[UntypedRow]: rows = await expdb.execute( text( @@ -113,12 +113,12 @@ async def get_input_for_task_type( WHERE `ttid`=:ttid AND `io`='input' """, ), - parameters={"ttid": task_type_id}, + params={"ttid": task_type_id}, ) return rows.all() -async def get_input_for_task(task_id: Identifier, expdb: AsyncConnection) -> Sequence[UntypedRow]: +async def get_input_for_task(task_id: Identifier, expdb: AsyncSession) -> Sequence[UntypedRow]: rows = await expdb.execute( text( """ @@ -127,14 +127,14 @@ async def get_input_for_task(task_id: Identifier, expdb: AsyncConnection) -> Seq WHERE task_id = :task_id """, ), - parameters={"task_id": task_id}, + params={"task_id": task_id}, ) return rows.all() async def get_task_type_inout_with_template( task_type: Identifier, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> Sequence[UntypedRow]: rows = await expdb.execute( text( @@ -144,7 +144,7 @@ async def get_task_type_inout_with_template( WHERE `ttid`=:ttid AND `template_api` IS NOT NULL """, ), - parameters={"ttid": task_type}, + params={"ttid": task_type}, ) return rows.all() diff --git a/src/database/users.py b/src/database/users.py index 9924cbba..4c1a64e2 100644 --- a/src/database/users.py +++ b/src/database/users.py @@ -11,7 +11,7 @@ from core.types import Identifier if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession api_key_pattern = re.compile(r"^[0-9a-fA-F]{32}$") @@ -45,7 +45,7 @@ class UserGroup(IntEnum): async def get_user( *, - connection: AsyncConnection, + session: AsyncSession, api_key: APIKey | None = None, user_id: Identifier | None = None, ) -> User | None: @@ -69,9 +69,9 @@ async def get_user( LIMIT 1 """ - result = await connection.execute( + result = await session.execute( text(query), - parameters={"api_key": api_key, "user_id": user_id}, + params={"api_key": api_key, "user_id": user_id}, ) row = result.one_or_none() if row: @@ -79,7 +79,7 @@ async def get_user( user_id=row.id, first_name=row.first_name, last_name=row.last_name, - _database=connection, + _database=session, ) return None @@ -87,9 +87,9 @@ async def get_user( async def get_user_groups_for( *, user_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> list[UserGroup]: - row = await connection.execute( + row = await session.execute( text( """ SELECT group_id @@ -97,7 +97,7 @@ async def get_user_groups_for( WHERE user_id = :user_id """, ), - parameters={"user_id": user_id}, + params={"user_id": user_id}, ) rows = row.all() return [UserGroup(group) for (group,) in rows] @@ -108,7 +108,7 @@ class User: user_id: Identifier first_name: str = "" last_name: str = "" - _database: AsyncConnection | None = None + _database: AsyncSession | None = None _groups: list[UserGroup] | None = None def __post_init__(self) -> None: @@ -124,8 +124,8 @@ def full_name(self) -> str: return f"{self.first_name} {self.last_name}".strip() @classmethod - async def fetch(cls, api_key: APIKey, user_db: AsyncConnection) -> Self | None: - user = await get_user(api_key=api_key, connection=user_db) + async def fetch(cls, api_key: APIKey, user_db: AsyncSession) -> Self | None: + user = await get_user(api_key=api_key, session=user_db) if user is not None: return cls( user_id=user.user_id, @@ -140,12 +140,12 @@ async def get_groups(self) -> list[UserGroup]: return self._groups if self._database is None: - msg = "`get_groups` can only be used when `connection` is provided on instantiation." + msg = "`get_groups` can only be used when `session` is provided on instantiation." raise RuntimeError(msg) self._groups = await get_user_groups_for( user_id=self.user_id, - connection=self._database, + session=self._database, ) return self._groups @@ -153,15 +153,15 @@ async def is_admin(self) -> bool: return UserGroup.ADMIN in await self.get_groups() -async def exists_by_id(*, user_id: Identifier, connection: AsyncConnection) -> bool: - row = await connection.execute( +async def exists_by_id(*, user_id: Identifier, session: AsyncSession) -> bool: + row = await session.execute( text("SELECT 1 FROM users WHERE id = :user_id LIMIT 1"), - parameters={"user_id": user_id}, + params={"user_id": user_id}, ) return row.one_or_none() is not None -async def has_user_references(*, user_id: Identifier, expdb: AsyncConnection) -> bool: +async def has_user_references(*, user_id: Identifier, expdb: AsyncSession) -> bool: """Return ``True`` if any ``expdb`` row still references ``user_id``.""" row = await expdb.execute( text( @@ -185,18 +185,18 @@ async def has_user_references(*, user_id: Identifier, expdb: AsyncConnection) -> ) AS has_refs """, ), - parameters={"uid": user_id}, + params={"uid": user_id}, ) return bool(row.scalar_one()) -async def delete_user_rows(*, user_id: Identifier, userdb: AsyncConnection) -> None: +async def delete_user_rows(*, user_id: Identifier, userdb: AsyncSession) -> None: """Remove group memberships then the user row (openml user database).""" await userdb.execute( text("DELETE FROM users_groups WHERE user_id = :user_id"), - parameters={"user_id": user_id}, + params={"user_id": user_id}, ) await userdb.execute( text("DELETE FROM users WHERE id = :user_id"), - parameters={"user_id": user_id}, + params={"user_id": user_id}, ) diff --git a/src/routers/datasets.py b/src/routers/datasets.py index dafaa43c..530234cf 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -51,10 +51,10 @@ from database.users import User from routers.dependencies import ( Pagination, - expdb_connection, + expdb_session, fetch_user, fetch_user_or_raise, - userdb_connection, + userdb_session, ) from routers.schemas.core import TagInfo from routers.schemas.datasets import ( @@ -67,7 +67,7 @@ if TYPE_CHECKING: from sqlalchemy.engine import Row - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/datasets", tags=["datasets"]) @@ -95,11 +95,11 @@ async def tag_dataset( data_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[str, dict[str, Any]]: """Add a tag to the dataset, this tag is publicly visible to all users.""" try: - await database.datasets.tag(data_id, tag, user_id=user.user_id, connection=expdb_db) + await database.datasets.tag(data_id, tag, user_id=user.user_id, session=expdb_db) except ForeignKeyConstraintError: msg = f"Dataset {data_id} not found." raise DatasetNotFoundError(msg, code=472) from None @@ -121,11 +121,11 @@ async def untag_dataset_like_php( data_id: Annotated[Identifier, Body()], tag: Annotated[TagString, Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[Literal["data_untag"], TagInfo]: """Remove a tag from the dataset with a response similar to the old PHP server.""" await untag_dataset(data_id, tag, user, expdb_db) - tags = await database.datasets.get_tags_for(dataset_id=data_id, connection=expdb_db) + tags = await database.datasets.get_tags_for(dataset_id=data_id, session=expdb_db) tag_info: TagInfo = {"id": str(data_id)} if len(tags) == 1: tag_info["tag"] = tags[0] @@ -139,7 +139,7 @@ async def untag_dataset( identifier: Identifier, tag: Annotated[TagString, Query()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session)], ) -> None: """Remove a tag that you added to a dataset, or from a dataset you uploaded.""" dataset_tag = await database.datasets.get_tag(identifier, tag, expdb_db) @@ -186,7 +186,7 @@ def _quality_clause(quality: str, range_: str | None) -> str: @router.post(path="/list", description="Provided for convenience, same as `GET` endpoint.") @router.get(path="/list") async def list_datasets( # noqa: PLR0913, C901 - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session)], pagination: Annotated[Pagination, Body(default_factory=Pagination)], data_name: Annotated[CasualString128 | None, Body()] = None, tag: Annotated[TagString | None, Body()] = None, @@ -296,7 +296,7 @@ async def list_datasets( # noqa: PLR0913, C901 matching_filter.bindparams(bindparam("data_ids", expanding=True)) result = await expdb_db.execute( matching_filter, - parameters=parameters, + params=parameters, ) rows = result.all() datasets: dict[int, dict[str, Any]] = { @@ -331,7 +331,7 @@ async def list_datasets( # noqa: PLR0913, C901 qualities_by_dataset = await database.qualities.get_for_datasets( dataset_ids=datasets.keys(), quality_names=qualities_to_show, - connection=expdb_db, + session=expdb_db, ) for did, qualities in qualities_by_dataset.items(): datasets[did]["quality"] = qualities @@ -348,13 +348,13 @@ class ProcessingInformation(NamedTuple): async def _get_processing_information( dataset_id: Identifier, - connection: AsyncConnection, + session: AsyncSession, ) -> ProcessingInformation: """Return processing information, if any. Otherwise, all fields `None`.""" if not ( data_processed := await database.datasets.get_latest_processing_update( dataset_id, - connection, + session, ) ): return ProcessingInformation(date=None, warning=None, error=None) @@ -368,7 +368,7 @@ async def _get_processing_information( async def _get_dataset_raise_otherwise( dataset_id: Identifier, user: User | None, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> Row[Any]: """Fetch the dataset from the database if it exists and the user has permissions. @@ -388,7 +388,7 @@ async def _get_dataset_raise_otherwise( @router.get("/features/{dataset_id}", response_model_exclude_none=True) async def get_dataset_features( dataset_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> list[Feature]: """Return metadata for each feature (column) in the dataset.""" @@ -405,7 +405,7 @@ async def get_dataset_features( feature.nominal_values = await database.datasets.get_feature_values( dataset_id, feature_index=feature.index, - connection=expdb, + session=expdb, ) if not features: @@ -434,7 +434,7 @@ async def update_dataset_status( dataset_id: Annotated[Identifier, Body()], status: Annotated[Literal[DatasetStatus.ACTIVE, DatasetStatus.DEACTIVATED], Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[str, str | int]: """Update the status of the dataset. Can be used to deactivate a dataset.""" dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb) @@ -464,7 +464,7 @@ async def update_dataset_status( dataset_id, status, user_id=user.user_id, - connection=expdb, + session=expdb, ) elif current_status == DatasetStatus.DEACTIVATED: await database.datasets.remove_deactivated_status(dataset_id, expdb) @@ -487,8 +487,8 @@ async def update_dataset_status( ) async def get_dataset( dataset_id: Identifier, - user_db: Annotated[AsyncConnection, Depends(userdb_connection)], - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], + user_db: Annotated[AsyncSession, Depends(userdb_session)], + expdb_db: Annotated[AsyncSession, Depends(expdb_session)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> DatasetMetadata: """Get the user-provided metadata for a dataset.""" @@ -496,7 +496,7 @@ async def get_dataset( if not ( dataset_file := await database.datasets.get_file( file_id=dataset.file_id, - connection=user_db, + session=user_db, ) ): msg = f"No data file found for dataset {dataset_id}." diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index f0706ef0..f8bbd1f8 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -4,7 +4,7 @@ """ from collections.abc import AsyncGenerator, AsyncIterator -from typing import TYPE_CHECKING, Annotated +from typing import Annotated from fastapi import Depends from loguru import logger @@ -15,9 +15,6 @@ from database.engine import expdb_database, user_database from database.users import APIKey, User -if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection - async def expdb_connection() -> AsyncIterator[AsyncConnection]: engine = expdb_database() @@ -31,16 +28,21 @@ async def userdb_connection() -> AsyncIterator[AsyncConnection]: yield connection -async def expdb_session( - connection: Annotated[AsyncConnection, Depends(expdb_connection)], -) -> AsyncIterator[AsyncSession]: - async with AsyncSession(connection) as session: +async def expdb_session() -> AsyncIterator[AsyncSession]: + engine = expdb_database() + async with AsyncSession(engine) as session, session.begin(): + yield session + + +async def userdb_session() -> AsyncIterator[AsyncSession]: + engine = user_database() + async with AsyncSession(engine) as session, session.begin(): yield session async def fetch_user( api_key: APIKey | None = None, - user_data: Annotated[AsyncConnection | None, Depends(userdb_connection)] = None, + user_data: Annotated[AsyncSession | None, Depends(userdb_session)] = None, ) -> AsyncGenerator[User | None]: if not (api_key and user_data): yield None diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index 33c942e9..48904f5a 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -5,18 +5,18 @@ from fastapi import APIRouter, Depends import database.evaluations -from routers.dependencies import expdb_connection +from routers.dependencies import expdb_session from routers.schemas.tasks import EstimationProcedure if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/estimationprocedure", tags=["estimationprocedure"]) @router.get("/list", response_model_exclude_none=True) async def get_estimation_procedures( - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> list[EstimationProcedure]: """Return a list with descriptions of estimation procedures. diff --git a/src/routers/evaluations.py b/src/routers/evaluations.py index 65cb4506..a0a73be6 100644 --- a/src/routers/evaluations.py +++ b/src/routers/evaluations.py @@ -5,17 +5,17 @@ from fastapi import APIRouter, Depends import database.evaluations -from routers.dependencies import expdb_connection +from routers.dependencies import expdb_session if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/evaluationmeasure", tags=["evaluationmeasure"]) @router.get("/list") async def get_evaluation_measures( - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> list[str]: """Return a list of evaluation measure names. @@ -27,6 +27,6 @@ async def get_evaluation_measures( """ functions = await database.evaluations.get_math_functions( function_type="EvaluationFunction", - connection=expdb, + session=expdb, ) return [function.name for function in functions] diff --git a/src/routers/flows.py b/src/routers/flows.py index 4380efbc..2e3bd596 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -9,11 +9,11 @@ from core.conversions import str_to_num from core.errors import FlowNotFoundError from core.types import Identifier -from routers.dependencies import expdb_connection +from routers.dependencies import expdb_session from routers.schemas.flows import Flow, Parameter, Subflow if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/flows", tags=["flows"]) @@ -22,7 +22,7 @@ async def flow_exists( name: str, external_version: str, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[Literal["flow_id"], int]: """Check if a Flow with the name and version exists, if so, return the flow id.""" flow = await database.flows.get_by_name( @@ -39,7 +39,7 @@ async def flow_exists( @router.get("/{flow_id}") async def get_flow( flow_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> Flow: """Get a Flow by its identifier.""" flow = await database.flows.get(flow_id, expdb) diff --git a/src/routers/qualities.py b/src/routers/qualities.py index 9b21d23f..24d33892 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -19,21 +19,21 @@ ) from core.types import Identifier from database.users import User -from routers.dependencies import expdb_connection, fetch_user +from routers.dependencies import expdb_session, fetch_user from routers.schemas.datasets import Quality if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/datasets", tags=["datasets"]) @router.get("/qualities/list") async def list_qualities( - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[Literal["data_qualities_list"], dict[Literal["quality"], list[str]]]: """List names of all computed qualities (dataset metafeatures).""" - qualities = await database.qualities.list_all_qualities(connection=expdb) + qualities = await database.qualities.list_all_qualities(session=expdb) return { "data_qualities_list": { "quality": qualities, @@ -45,7 +45,7 @@ async def list_qualities( async def get_qualities( dataset_id: Identifier, user: Annotated[User | None, Depends(fetch_user)], - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> list[Quality]: """Get computed qualities (metafeatures) for a dataset.""" dataset = await database.datasets.get(dataset_id, expdb) diff --git a/src/routers/runs.py b/src/routers/runs.py index 4613df26..e7904f75 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -16,7 +16,7 @@ from core.types import Identifier from database.models.base import UntypedRow from database.models.setups import Setup -from routers.dependencies import expdb_connection, expdb_session, userdb_connection +from routers.dependencies import expdb_session, userdb_session from routers.schemas.runs import ( EvaluationScore, InputDataset, @@ -29,7 +29,7 @@ ) if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/run", tags=["run"]) @@ -37,7 +37,7 @@ @router.get("/trace/{run_id}") async def get_run_trace( run_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> RunTrace: """Get trace data for a run by run ID.""" if not await database.runs.exist(run_id, expdb): @@ -82,9 +82,8 @@ class RunContext: async def _load_run_context( run: UntypedRow, - expdb: AsyncConnection, - userdb: AsyncConnection, - expdb_session: AsyncSession, + expdb: AsyncSession, + userdb: AsyncSession, engine_ids: list[int], ) -> RunContext: ( @@ -100,14 +99,14 @@ async def _load_run_context( ) = cast( "tuple[Any, list[str], list[UntypedRow], list[UntypedRow], list[UntypedRow], str | None, str | None, Setup | None, list[UntypedRow]]", # noqa: E501 await asyncio.gather( - database.users.get_user(user_id=run.uploader, connection=userdb), + database.users.get_user(user_id=run.uploader, session=userdb), database.runs.get_tags(run.rid, expdb), database.runs.get_input_data(run.rid, expdb), database.runs.get_output_files(run.rid, expdb), database.runs.get_evaluations(run.rid, expdb, evaluation_engine_ids=engine_ids), database.tasks.get_task_type_name(run.task_id, expdb), database.tasks.get_task_evaluation_measure(run.task_id, expdb), - database.setups.get(run.setup, expdb_session), + database.setups.get(run.setup, expdb), database.setups.get_parameters(run.setup, expdb), ), ) @@ -145,9 +144,8 @@ def _normalise_value(v: object) -> object: @router.get("/{run_id}", response_model_exclude_none=True) async def get_run( run_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], - userdb: Annotated[AsyncConnection, Depends(userdb_connection)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], + userdb: Annotated[AsyncSession, Depends(userdb_session)], ) -> Run: """Get full metadata for a run by ID.""" # Authentication is not required because runs are always public. @@ -159,7 +157,7 @@ async def get_run( raise RunNotFoundError(msg, code=236) engine_ids: list[int] = config.get_config().development.run_evaluation_engine_ids - ctx = await _load_run_context(run, expdb, userdb, expdb_session, engine_ids) + ctx = await _load_run_context(run, expdb, userdb, engine_ids) flow = await database.flows.get(ctx.setup.flow_id, expdb) if ctx.setup else None evaluations = _build_evaluations(ctx.evaluation_rows) diff --git a/src/routers/setups.py b/src/routers/setups.py index aadd7224..04373d17 100644 --- a/src/routers/setups.py +++ b/src/routers/setups.py @@ -15,11 +15,11 @@ from core.types import Identifier, TagString from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError from database.users import User -from routers.dependencies import expdb_connection, expdb_session, fetch_user_or_raise +from routers.dependencies import expdb_session, fetch_user_or_raise from routers.schemas.setups import SetupParameters, SetupResponse if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/setup", tags=["setup"]) @@ -27,7 +27,6 @@ @router.get(path="/{setup_id}", response_model_exclude_none=True) async def get_setup( setup_id: Annotated[Identifier, Path()], - expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> SetupResponse: """Get setup by id.""" @@ -36,7 +35,7 @@ async def get_setup( msg = f"Setup {setup_id} not found." raise SetupNotFoundError(msg, code=281) - setup_parameters = await database.setups.get_parameters(setup_id, expdb_db) + setup_parameters = await database.setups.get_parameters(setup_id, expdb_session) params_model = SetupParameters( setup_id=setup_id, diff --git a/src/routers/study.py b/src/routers/study.py index 5f0f5d1f..23a6d60d 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -21,12 +21,12 @@ from core.types import Identifier from database.models.base import UntypedRow from database.users import User -from routers.dependencies import expdb_connection, fetch_user, fetch_user_or_raise +from routers.dependencies import expdb_session, fetch_user, fetch_user_or_raise from routers.schemas.core import Visibility from routers.schemas.study import CreateStudy, Study, StudyStatus, StudyType if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/studies", tags=["studies"]) @@ -34,7 +34,7 @@ async def _get_study_raise_otherwise( id_or_alias: Identifier | str, user: User | None, - expdb: AsyncConnection, + expdb: AsyncSession, ) -> UntypedRow: search_by_id = isinstance(id_or_alias, int) or id_or_alias.isdigit() if search_by_id: @@ -71,7 +71,7 @@ async def attach_to_study( study_id: Annotated[Identifier, Body()], entity_ids: Annotated[list[Identifier], Body()], user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> AttachDetachResponse: """Add runs or tasks to a study which is in preparation.""" if user is None: @@ -100,14 +100,14 @@ async def attach_to_study( task_ids=entity_ids, study_id=study_id, user=user, - connection=expdb, + session=expdb, ) else: await database.studies.attach_runs( run_ids=entity_ids, study_id=study_id, user=user, - connection=expdb, + session=expdb, ) except ValueError as e: msg = str(e) @@ -125,7 +125,7 @@ async def attach_to_study( async def create_study( study: CreateStudy, user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[Literal["study_id"], int]: """Create a new study.""" if study.main_entity_type == StudyType.RUN and study.tasks: @@ -161,7 +161,7 @@ async def create_study( @router.get("/{alias_or_id}") async def get_study( alias_or_id: Identifier | str, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> Study: """Get a study by id or alias.""" diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 7c07b116..cb91c997 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -25,11 +25,11 @@ from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError from database.models.base import UntypedRow from database.users import User -from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise +from routers.dependencies import Pagination, expdb_session, fetch_user_or_raise from routers.schemas.tasks import Task if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/tasks", tags=["tasks"]) @@ -75,7 +75,7 @@ async def fill_template( template: str, task: UntypedRow, task_inputs: dict[str, str | int], - connection: AsyncConnection, + session: AsyncSession, ) -> dict[str, JSON]: """Fill in the XML template as used for task descriptions and return the result. @@ -133,7 +133,7 @@ async def fill_template( task, task_inputs, fetched_data={}, - connection=connection, + session=session, ), ) @@ -143,16 +143,16 @@ async def _fill_json_template( # noqa: C901 task: UntypedRow, task_inputs: dict[str, str | int], fetched_data: dict[str, str], - connection: AsyncConnection, + session: AsyncSession, ) -> JSON: if isinstance(template, dict): return { - k: await _fill_json_template(v, task, task_inputs, fetched_data, connection) + k: await _fill_json_template(v, task, task_inputs, fetched_data, session) for k, v in template.items() } if isinstance(template, list): return [ - await _fill_json_template(v, task, task_inputs, fetched_data, connection) + await _fill_json_template(v, task, task_inputs, fetched_data, session) for v in template ] if not isinstance(template, str): @@ -171,7 +171,7 @@ async def _fill_json_template( # noqa: C901 (field,) = match.groups() if field not in fetched_data: table, _ = field.split(".") - result = await connection.execute( + result = await session.execute( text( f""" SELECT * @@ -181,7 +181,7 @@ async def _fill_json_template( # noqa: C901 ), # Not sure how parametrize table names, as the parametrization adds # quotes which is not legal. - parameters={"id_": int(task_inputs[table])}, + params={"id_": int(task_inputs[table])}, ) rows = result.mappings() row_data = next(rows, None) @@ -260,7 +260,7 @@ def _quality_clause(quality: str, range_: str | None) -> str: @router.post(path="/list", description="Provided for convenience, same as `GET` endpoint.") @router.get(path="/list") async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], pagination: Annotated[Pagination, Body(default_factory=Pagination)], task_type_id: Annotated[Identifier | None, Body(description="Filter by task type id.")] = None, tag: Annotated[TagString | None, Body()] = None, @@ -369,7 +369,7 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 if data_id is not None: main_query = main_query.bindparams(bindparam("data_ids", expanding=True)) - result = await expdb.execute(main_query, parameters=parameters) + result = await expdb.execute(main_query, params=parameters) rows = result.mappings().all() if not rows: @@ -417,15 +417,15 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 inputs_result, qualities_result, tags_result = await asyncio.gather( expdb.execute( inputs_query, - parameters={"task_ids": task_ids, "basic_inputs": BASIC_TASK_INPUTS}, + params={"task_ids": task_ids, "basic_inputs": BASIC_TASK_INPUTS}, ), expdb.execute( qualities_query, - parameters={"dataset_ids": dataset_ids, "quality_names": QUALITIES_TO_SHOW}, + params={"dataset_ids": dataset_ids, "quality_names": QUALITIES_TO_SHOW}, ), expdb.execute( tags_query, - parameters={"task_ids": task_ids}, + params={"task_ids": task_ids}, ), ) @@ -453,8 +453,7 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 @router.get("/{task_id}") async def get_task( task_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], - expdb_session: Annotated[AsyncSession, Depends(expdb_session)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> Task: """Get a task by identifier.""" if not (task := await database.tasks.get(task_id, expdb)): @@ -467,7 +466,7 @@ async def get_task( task_input_rows, ttios, tags = await asyncio.gather( database.tasks.get_input_for_task(task_id, expdb), database.tasks.get_task_type_inout_with_template(task_type.ttid, expdb), - database.tasks.get_tags(task_id, expdb_session), + database.tasks.get_tags(task_id, expdb), ) task_inputs = { row.input: int(row.value) if row.value.isdigit() else row.value for row in task_input_rows diff --git a/src/routers/tasktype.py b/src/routers/tasktype.py index 575c78ea..a1a4c5aa 100644 --- a/src/routers/tasktype.py +++ b/src/routers/tasktype.py @@ -13,11 +13,11 @@ from core.types import Identifier from database.tasks import get_input_for_task_type, get_task_types from database.tasks import get_task_type as db_get_task_type -from routers.dependencies import expdb_connection +from routers.dependencies import expdb_session if TYPE_CHECKING: from sqlalchemy.engine import Row - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession router = APIRouter(prefix="/tasktype", tags=["tasks"]) @@ -37,7 +37,7 @@ def _normalize_task_type(task_type: Row[Any]) -> dict[str, str | None | list[Any @router.get(path="/list") async def list_task_types( - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[ Literal["task_types"], dict[Literal["task_type"], list[dict[str, str | None | list[Any]]]], @@ -52,7 +52,7 @@ async def list_task_types( @router.get(path="/{task_type_id}") async def get_task_type( task_type_id: Identifier, - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[Literal["task_type"], dict[str, str | None | list[str] | list[dict[str, str]]]]: """Return a detailed description for the given task type, including expected inputs.""" task_type_record = await db_get_task_type(task_type_id, expdb) diff --git a/src/routers/users.py b/src/routers/users.py index 14a900e7..aba293e0 100644 --- a/src/routers/users.py +++ b/src/routers/users.py @@ -6,12 +6,12 @@ from fastapi import APIRouter, Depends, Path, Response from loguru import logger from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncConnection # noqa: TC002 used at runtime by FastAPI Depends +from sqlalchemy.ext.asyncio import AsyncSession # noqa: TC002 used at runtime by FastAPI Depends import database.users from core.errors import AccountHasResourcesError, ForbiddenError, UserNotFoundError from database.users import User -from routers.dependencies import expdb_connection, fetch_user_or_raise, userdb_connection +from routers.dependencies import expdb_session, fetch_user_or_raise, userdb_session _ACCOUNT_HAS_RESOURCES_MSG = ( "Cannot delete this account while records still reference the user " @@ -36,8 +36,8 @@ async def delete_user_account( user_id: Annotated[int, Path(description="Numeric user id to delete.", gt=0)], current_user: Annotated[User, Depends(fetch_user_or_raise)], - expdb: Annotated[AsyncConnection, Depends(expdb_connection)], - userdb: Annotated[AsyncConnection, Depends(userdb_connection)], + expdb: Annotated[AsyncSession, Depends(expdb_session)], + userdb: Annotated[AsyncSession, Depends(userdb_session)], ) -> Response: """Delete the user account if they have no associated resources. @@ -51,7 +51,7 @@ async def delete_user_account( msg = "You may only delete your own user account." raise ForbiddenError(msg) - if not await database.users.exists_by_id(user_id=user_id, connection=userdb): + if not await database.users.exists_by_id(user_id=user_id, session=userdb): msg = f"User {user_id} not found." raise UserNotFoundError(msg) diff --git a/tests/conftest.py b/tests/conftest.py index ea893f68..dbee65ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,10 @@ from core.types import Identifier from database.engine import expdb_database, user_database from main import create_api -from routers.dependencies import expdb_connection, userdb_connection +from routers.dependencies import ( + expdb_session as expdb_session_dep, + userdb_session as userdb_session_dep, +) from tests.users import OWNER_USER if TYPE_CHECKING: @@ -45,7 +48,7 @@ async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncConnecti @contextlib.asynccontextmanager async def temporary_records( - connection: AsyncConnection, + session: AsyncSession, insert_queries: Iterable[tuple[str, dict[str, Any] | None]], delete_queries: Iterable[tuple[str, dict[str, Any] | None]], *, @@ -53,17 +56,17 @@ async def temporary_records( ) -> AsyncIterator[None]: """Execute insert queries on enter and their corresponding delete queries on exit.""" for query, parameters in insert_queries: - await connection.execute(text(query), parameters=parameters) + await session.execute(text(query), params=parameters) if persist: - await connection.commit() + await session.commit() try: yield finally: for query, parameters in delete_queries: - await connection.execute(text(query), parameters=parameters) + await session.execute(text(query), params=parameters) if persist: - await connection.commit() + await session.commit() @pytest.fixture @@ -88,6 +91,13 @@ async def user_test() -> AsyncIterator[AsyncConnection]: yield connection +@pytest.fixture +async def userdb_session(user_test: AsyncConnection) -> AsyncIterator[AsyncSession]: + async with AsyncSession(user_test) as session: + yield session + await session.rollback() + + # The PHP API fixture can be session scoped since they do not need access to # function-scoped database transactions. @pytest.fixture(scope="session") @@ -127,14 +137,16 @@ async def py_api( # expdb_test.execute(...) # write some data # noqa: ERA001 # py_api.get(...) # read that data # noqa: ERA001 - async def override_expdb() -> AsyncIterator[AsyncConnection]: - yield expdb_test + async def override_expdb() -> AsyncIterator[AsyncSession]: + async with AsyncSession(expdb_test) as session: + yield session - async def override_userdb() -> AsyncIterator[AsyncConnection]: - yield user_test + async def override_userdb() -> AsyncIterator[AsyncSession]: + async with AsyncSession(user_test) as session: + yield session - app.dependency_overrides[expdb_connection] = override_expdb - app.dependency_overrides[userdb_connection] = override_userdb + app.dependency_overrides[expdb_session_dep] = override_expdb + app.dependency_overrides[userdb_session_dep] = override_userdb async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), @@ -143,8 +155,8 @@ async def override_userdb() -> AsyncIterator[AsyncConnection]: ) as client: yield client - app.dependency_overrides[expdb_connection] = expdb_connection - app.dependency_overrides[userdb_connection] = userdb_connection + app.dependency_overrides[expdb_session_dep] = expdb_session + app.dependency_overrides[userdb_session_dep] = userdb_session @pytest.fixture @@ -188,7 +200,7 @@ def _get() -> Identifier: @pytest.fixture async def task_factory( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> TaskFactory: async def create_task( *, @@ -198,11 +210,11 @@ async def create_task( ) -> Task: task_id = task_id or _identifier_factory() - await expdb_test.execute( + await expdb_session.execute( text(""" INSERT INTO task (task_id, ttid, creator) VALUES (:task_id, :ttid, :creator); """), - parameters={"task_id": task_id, "ttid": task_type, "creator": creator}, + params={"task_id": task_id, "ttid": task_type, "creator": creator}, ) return Task(task_id, task_type, creator) @@ -217,13 +229,13 @@ def __call__( @pytest.fixture async def dataset_factory( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> DatasetFactory: async def create_dataset( *, dataset_id: Identifier | None = None, creator: Identifier = OWNER_USER.user_id ) -> Identifier: dataset_id = dataset_id or _identifier_factory() - await expdb_test.execute( + await expdb_session.execute( text(""" INSERT INTO dataset (did, uploader, name, version, format, upload_date, licence, url, visibility) @@ -231,7 +243,7 @@ async def create_dataset( (:dataset_id, :creator, :name, 'dataset-version', 'dataset-format', :now, 'public', 'dataset-url', 'public'); """), - parameters={ + params={ "dataset_id": dataset_id, "creator": creator, "now": datetime.datetime.now(tz=datetime.UTC), @@ -252,8 +264,8 @@ class Flow(NamedTuple): @pytest.fixture -async def flow(expdb_test: AsyncConnection) -> Flow: - await expdb_test.execute( +async def flow(expdb_session: AsyncSession) -> Flow: + await expdb_session.execute( text( """ INSERT INTO implementation(fullname,name,version,external_version,uploadDate) @@ -261,34 +273,34 @@ async def flow(expdb_test: AsyncConnection) -> Flow: """, ), ) - result = await expdb_test.execute(text("""SELECT LAST_INSERT_ID();""")) + result = await expdb_session.execute(text("""SELECT LAST_INSERT_ID();""")) (flow_id,) = result.one() return Flow(id=flow_id, name="name", external_version="external_version") @pytest.fixture -async def persisted_flow(flow: Flow, expdb_test: AsyncConnection) -> AsyncIterator[Flow]: - await expdb_test.commit() +async def persisted_flow(flow: Flow, expdb_session: AsyncSession) -> AsyncIterator[Flow]: + await expdb_session.commit() yield flow # We want to ensure the commit below does not accidentally persist new # data to the database. - await expdb_test.rollback() + await expdb_session.rollback() - await expdb_test.execute( + await expdb_session.execute( text( """ DELETE FROM implementation WHERE id = :flow_id """, ), - parameters={"flow_id": flow.id}, + params={"flow_id": flow.id}, ) - await expdb_test.commit() + await expdb_session.commit() @pytest.fixture def temporary_tags( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> Callable[..., contextlib.AbstractAsyncContextManager[None]]: @contextlib.asynccontextmanager async def _temporary_tags( @@ -317,7 +329,7 @@ async def _temporary_tags( for tag in tags ] async with temporary_records( - connection=expdb_test, + session=expdb_session, insert_queries=insert_queries, delete_queries=delete_queries, persist=persist, diff --git a/tests/database/flows_test.py b/tests/database/flows_test.py index 6e345773..82ec06a2 100644 --- a/tests/database/flows_test.py +++ b/tests/database/flows_test.py @@ -4,20 +4,20 @@ from tests.conftest import Flow if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession -async def test_database_flow_exists(flow: Flow, expdb_test: AsyncConnection) -> None: - retrieved_flow = await database.flows.get_by_name(flow.name, flow.external_version, expdb_test) +async def test_database_flow_exists(flow: Flow, expdb_session: AsyncSession) -> None: + retrieved_flow = await database.flows.get_by_name(flow.name, flow.external_version, expdb_session) assert retrieved_flow is not None assert retrieved_flow.id == flow.id # when using actual ORM, can instead ensure _all_ fields match. -async def test_database_flow_exists_returns_none_if_no_match(expdb_test: AsyncConnection) -> None: +async def test_database_flow_exists_returns_none_if_no_match(expdb_session: AsyncSession) -> None: retrieved_flow = await database.flows.get_by_name( name="foo", external_version="bar", - expdb=expdb_test, + expdb=expdb_session, ) assert retrieved_flow is None diff --git a/tests/database/runs_test.py b/tests/database/runs_test.py index 80f2151c..b7d03866 100644 --- a/tests/database/runs_test.py +++ b/tests/database/runs_test.py @@ -1,6 +1,6 @@ """Tests for database layer of runs.""" -from sqlalchemy.ext.asyncio import AsyncConnection # noqa: TC002 +from sqlalchemy.ext.asyncio import AsyncSession # noqa: TC002 import database.runs import database.tasks @@ -17,9 +17,9 @@ _PREDICTIONS_FILE_ID = 183 -async def test_db_get_run_exists(expdb_test: AsyncConnection) -> None: +async def test_db_get_run_exists(expdb_session: AsyncSession) -> None: """database.runs.get returns a row for run 24.""" - row = await database.runs.get(_RUN_ID, expdb_test) + row = await database.runs.get(_RUN_ID, expdb_session) assert row is not None assert row.rid == _RUN_ID assert row.uploader == _RUN_UPLOADER_ID @@ -28,80 +28,80 @@ async def test_db_get_run_exists(expdb_test: AsyncConnection) -> None: assert row.error_message is None # no error for this run -async def test_db_get_run_missing(expdb_test: AsyncConnection) -> None: +async def test_db_get_run_missing(expdb_session: AsyncSession) -> None: """database.runs.get returns None for a non-existent run.""" - row = await database.runs.get(_MISSING_RUN_ID, expdb_test) + row = await database.runs.get(_MISSING_RUN_ID, expdb_session) assert row is None -async def test_db_exist_true(expdb_test: AsyncConnection) -> None: +async def test_db_exist_true(expdb_session: AsyncSession) -> None: """database.runs.exist returns True for run 24.""" - assert await database.runs.exist(_RUN_ID, expdb_test) is True + assert await database.runs.exist(_RUN_ID, expdb_session) is True -async def test_db_exist_false(expdb_test: AsyncConnection) -> None: +async def test_db_exist_false(expdb_session: AsyncSession) -> None: """database.runs.exist returns False for a missing run.""" - assert await database.runs.exist(_MISSING_RUN_ID, expdb_test) is False + assert await database.runs.exist(_MISSING_RUN_ID, expdb_session) is False -async def test_db_get_tags(expdb_test: AsyncConnection) -> None: +async def test_db_get_tags(expdb_session: AsyncSession) -> None: """database.runs.get_tags returns expected tags for run 24.""" - tags = await database.runs.get_tags(_RUN_ID, expdb_test) + tags = await database.runs.get_tags(_RUN_ID, expdb_session) assert isinstance(tags, list) assert "openml-python" in tags -async def test_db_get_input_data(expdb_test: AsyncConnection) -> None: +async def test_db_get_input_data(expdb_session: AsyncSession) -> None: """database.runs.get_input_data returns did=20 (diabetes) for run 24.""" - rows = await database.runs.get_input_data(_RUN_ID, expdb_test) + rows = await database.runs.get_input_data(_RUN_ID, expdb_session) assert len(rows) >= 1 dids = [r.did for r in rows] assert _RUN_DATASET_ID in dids -async def test_db_get_output_files(expdb_test: AsyncConnection) -> None: +async def test_db_get_output_files(expdb_session: AsyncSession) -> None: """database.runs.get_output_files returns description and predictions files.""" - rows = await database.runs.get_output_files(_RUN_ID, expdb_test) + rows = await database.runs.get_output_files(_RUN_ID, expdb_session) file_map = {r.field: r.file_id for r in rows} assert file_map.get("description") == _DESCRIPTION_FILE_ID assert file_map.get("predictions") == _PREDICTIONS_FILE_ID -async def test_db_get_evaluations(expdb_test: AsyncConnection) -> None: +async def test_db_get_evaluations(expdb_session: AsyncSession) -> None: """database.runs.get_evaluations returns metrics including area_under_roc_curve.""" - rows = await database.runs.get_evaluations(_RUN_ID, expdb_test, evaluation_engine_ids=[1]) + rows = await database.runs.get_evaluations(_RUN_ID, expdb_session, evaluation_engine_ids=[1]) assert len(rows) > 0 names = {r.name for r in rows} assert "area_under_roc_curve" in names -async def test_db_get_evaluations_empty_engine_list(expdb_test: AsyncConnection) -> None: +async def test_db_get_evaluations_empty_engine_list(expdb_session: AsyncSession) -> None: """get_evaluations with no engine IDs returns an empty list (not an error).""" - rows = await database.runs.get_evaluations(_RUN_ID, expdb_test, evaluation_engine_ids=[]) + rows = await database.runs.get_evaluations(_RUN_ID, expdb_session, evaluation_engine_ids=[]) assert rows == [] -async def test_db_get_task_type(expdb_test: AsyncConnection) -> None: +async def test_db_get_task_type(expdb_session: AsyncSession) -> None: """database.runs.get_task_type returns 'Supervised Classification' for task 115.""" - task_type = await database.tasks.get_task_type_name(_RUN_TASK_ID, expdb_test) + task_type = await database.tasks.get_task_type_name(_RUN_TASK_ID, expdb_session) assert task_type == "Supervised Classification" -async def test_db_get_task_evaluation_measure_missing(expdb_test: AsyncConnection) -> None: +async def test_db_get_task_evaluation_measure_missing(expdb_session: AsyncSession) -> None: """get_task_evaluation_measure returns None (not '') when absent.""" - measure = await database.tasks.get_task_evaluation_measure(_RUN_TASK_ID, expdb_test) + measure = await database.tasks.get_task_evaluation_measure(_RUN_TASK_ID, expdb_session) assert measure is None -async def test_db_get_uploader_name(user_test: AsyncConnection) -> None: +async def test_db_get_uploader_name(userdb_session: AsyncSession) -> None: """database.runs.get_uploader_name returns 'Cynthia Glover' for user 1159.""" - user = await database.users.get_user(user_id=_RUN_UPLOADER_ID, connection=user_test) + user = await database.users.get_user(user_id=_RUN_UPLOADER_ID, session=userdb_session) assert user is not None assert user.full_name == "Cynthia Glover" assert user.user_id == _RUN_UPLOADER_ID -async def test_db_get_uploader_name_missing(user_test: AsyncConnection) -> None: +async def test_db_get_uploader_name_missing(userdb_session: AsyncSession) -> None: """get_uploader_name returns None for a non-existent user.""" - user = await database.users.get_user(user_id=_MISSING_USER_ID, connection=user_test) + user = await database.users.get_user(user_id=_MISSING_USER_ID, session=userdb_session) assert user is None diff --git a/tests/dependencies/fetch_user_test.py b/tests/dependencies/fetch_user_test.py index c9bfa08c..8a878f72 100644 --- a/tests/dependencies/fetch_user_test.py +++ b/tests/dependencies/fetch_user_test.py @@ -9,7 +9,7 @@ from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession @pytest.mark.parametrize( @@ -20,8 +20,8 @@ (ApiKey.SOME_USER, SOME_USER), ], ) -async def test_fetch_user(api_key: str, user: User, user_test: AsyncConnection) -> None: - async with aclosing(fetch_user(api_key, user_data=user_test)) as agen: +async def test_fetch_user(api_key: str, user: User, userdb_session: AsyncSession) -> None: + async with aclosing(fetch_user(api_key, user_data=userdb_session)) as agen: db_user = await anext(agen) assert isinstance(db_user, User) assert db_user.user_id == user.user_id @@ -33,9 +33,9 @@ async def test_fetch_user_no_key_no_user() -> None: assert await anext(agen) is None -async def test_fetch_user_invalid_key_raises(user_test: AsyncConnection) -> None: +async def test_fetch_user_invalid_key_raises(userdb_session: AsyncSession) -> None: with pytest.raises(AuthenticationFailedError): - async with aclosing(fetch_user(api_key=ApiKey.INVALID, user_data=user_test)) as agen: + async with aclosing(fetch_user(api_key=ApiKey.INVALID, user_data=userdb_session)) as agen: await anext(agen) diff --git a/tests/routers/dataset_tag_test.py b/tests/routers/dataset_tag_test.py index 8ff37edc..851e395b 100644 --- a/tests/routers/dataset_tag_test.py +++ b/tests/routers/dataset_tag_test.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession @pytest.mark.parametrize( @@ -43,51 +43,51 @@ async def test_dataset_tag_rejects_unauthorized(key: ApiKey, py_api: httpx.Async ids=["administrator", "non-owner", "owner"], ) async def test_dataset_tag( - user: User, expdb_test: AsyncConnection, dataset_factory: DatasetFactory + user: User, expdb_session: AsyncSession, dataset_factory: DatasetFactory ) -> None: dataset_id = await dataset_factory() tag = "test_tag" - result = await tag_dataset(data_id=dataset_id, tag=tag, user=user, expdb_db=expdb_test) + result = await tag_dataset(data_id=dataset_id, tag=tag, user=user, expdb_db=expdb_session) assert result == {"data_tag": {"id": str(dataset_id), "tag": [tag]}} - tags = await get_tags_for(dataset_id=dataset_id, connection=expdb_test) + tags = await get_tags_for(dataset_id=dataset_id, session=expdb_session) assert tag in tags @pytest.mark.mut async def test_dataset_tag_returns_existing_tags( - expdb_test: AsyncConnection, dataset_factory: DatasetFactory + expdb_session: AsyncSession, dataset_factory: DatasetFactory ) -> None: dataset_id = await dataset_factory() - await tag_dataset(data_id=dataset_id, tag="first", user=OWNER_USER, expdb_db=expdb_test) + await tag_dataset(data_id=dataset_id, tag="first", user=OWNER_USER, expdb_db=expdb_session) result = await tag_dataset( - data_id=dataset_id, tag="second", user=ADMIN_USER, expdb_db=expdb_test + data_id=dataset_id, tag="second", user=ADMIN_USER, expdb_db=expdb_session ) assert result == {"data_tag": {"id": str(dataset_id), "tag": ["first", "second"]}} @pytest.mark.mut async def test_dataset_tag_fails_if_tag_exists( - expdb_test: AsyncConnection, dataset_factory: DatasetFactory + expdb_session: AsyncSession, dataset_factory: DatasetFactory ) -> None: tag = "repeated_tag" dataset_id = await dataset_factory() - await tag_dataset(data_id=dataset_id, tag=tag, user=OWNER_USER, expdb_db=expdb_test) + await tag_dataset(data_id=dataset_id, tag=tag, user=OWNER_USER, expdb_db=expdb_session) with pytest.raises(TagAlreadyExistsError) as e: - await tag_dataset(data_id=dataset_id, tag=tag, user=ADMIN_USER, expdb_db=expdb_test) + await tag_dataset(data_id=dataset_id, tag=tag, user=ADMIN_USER, expdb_db=expdb_session) assert str(dataset_id) in e.value.detail assert tag in e.value.detail -async def test_dataset_tag_fails_if_dataset_does_not_exist(expdb_test: AsyncConnection) -> None: +async def test_dataset_tag_fails_if_dataset_does_not_exist(expdb_session: AsyncSession) -> None: dataset_id = 1_000_000 with pytest.raises(DatasetNotFoundError) as e: await tag_dataset( data_id=dataset_id, tag="foo", user=ADMIN_USER, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert str(dataset_id) in e.value.detail dataset_not_found_in_tag_endpoint = 472 diff --git a/tests/routers/dataset_untag_test.py b/tests/routers/dataset_untag_test.py index 7eca8893..c0673cbf 100644 --- a/tests/routers/dataset_untag_test.py +++ b/tests/routers/dataset_untag_test.py @@ -25,17 +25,17 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession async def test_dataset_untag_success( - py_api: httpx.AsyncClient, expdb_test: AsyncConnection + py_api: httpx.AsyncClient, expdb_session: AsyncSession ) -> None: dataset_id = 1 tag = "foo" - await expdb_test.execute( + await expdb_session.execute( text("INSERT INTO dataset_tag(id, tag, uploader) VALUES (:dataset_id, :tag, 2)"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) response = await py_api.delete( @@ -43,70 +43,70 @@ async def test_dataset_untag_success( ) assert response.status_code == HTTPStatus.NO_CONTENT - tag_present = await expdb_test.execute( + tag_present = await expdb_session.execute( text("SELECT 1 FROM dataset_tag WHERE id=:dataset_id AND tag=:tag"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) assert tag_present.scalar() is None -async def test_dataset_untag_tag_does_not_exist(expdb_test: AsyncConnection) -> None: +async def test_dataset_untag_tag_does_not_exist(expdb_session: AsyncSession) -> None: dataset_id = 1 tag = "foo" with pytest.raises(TagNotFoundError) as e: - await untag_dataset(dataset_id, tag, SOME_USER, expdb_test) + await untag_dataset(dataset_id, tag, SOME_USER, expdb_session) assert e.value.status_code == HTTPStatus.NOT_FOUND assert tag in e.value.detail assert str(dataset_id) in e.value.detail -async def test_dataset_untag_tag_not_owned(expdb_test: AsyncConnection) -> None: +async def test_dataset_untag_tag_not_owned(expdb_session: AsyncSession) -> None: dataset_id = 1 tag = "foo" - await expdb_test.execute( + await expdb_session.execute( text("INSERT INTO dataset_tag(id, tag, uploader) VALUES (:dataset_id, :tag, 1)"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) with pytest.raises(TagNotOwnedError) as e: - await untag_dataset(dataset_id, tag, SOME_USER, expdb_test) + await untag_dataset(dataset_id, tag, SOME_USER, expdb_session) assert e.value.status_code == HTTPStatus.FORBIDDEN assert tag in e.value.detail assert str(dataset_id) in e.value.detail - tag_present = await expdb_test.execute( + tag_present = await expdb_session.execute( text("SELECT 1 FROM dataset_tag WHERE id=:dataset_id AND tag=:tag"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) assert tag_present.scalar() == 1 -async def test_dataset_untag_admin_bypasses_ownership(expdb_test: AsyncConnection) -> None: +async def test_dataset_untag_admin_bypasses_ownership(expdb_session: AsyncSession) -> None: dataset_id = 1 tag = "foo" - await expdb_test.execute( + await expdb_session.execute( text("INSERT INTO dataset_tag(id, tag, uploader) VALUES (:dataset_id, :tag, 1)"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) - await untag_dataset(dataset_id, tag, ADMIN_USER, expdb_test) + await untag_dataset(dataset_id, tag, ADMIN_USER, expdb_session) - tag_present = await expdb_test.execute( + tag_present = await expdb_session.execute( text("SELECT 1 FROM dataset_tag WHERE id=:dataset_id AND tag=:tag"), - parameters={"dataset_id": dataset_id, "tag": tag}, + params={"dataset_id": dataset_id, "tag": tag}, ) assert tag_present.scalar() is None -async def test_dataset_untag_dataset_does_not_exist(expdb_test: AsyncConnection) -> None: +async def test_dataset_untag_dataset_does_not_exist(expdb_session: AsyncSession) -> None: dataset_id = 9_999_999 tag = "foo" with pytest.raises(DatasetNotFoundError) as e: - await untag_dataset(dataset_id, tag, SOME_USER, expdb_test) + await untag_dataset(dataset_id, tag, SOME_USER, expdb_session) assert e.value.status_code == HTTPStatus.NOT_FOUND assert tag in e.value.detail diff --git a/tests/routers/datasets_features_test.py b/tests/routers/datasets_features_test.py index df552604..a8063173 100644 --- a/tests/routers/datasets_features_test.py +++ b/tests/routers/datasets_features_test.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def test_get_features_via_api(py_api: httpx.AsyncClient) -> None: @@ -71,8 +71,8 @@ async def test_get_features_via_api(py_api: httpx.AsyncClient) -> None: ] -async def test_dataset_features_with_ontology(expdb_test: AsyncConnection) -> None: - features = await get_dataset_features(dataset_id=11, user=None, expdb=expdb_test) +async def test_dataset_features_with_ontology(expdb_session: AsyncSession) -> None: + features = await get_dataset_features(dataset_id=11, user=None, expdb=expdb_session) by_index = {f.index: f for f in features} assert by_index[1].ontology == ["https://en.wikipedia.org/wiki/Service_(motor_vehicle)"] assert by_index[2].ontology == [ @@ -86,28 +86,28 @@ async def test_dataset_features_with_ontology(expdb_test: AsyncConnection) -> No assert by_index[4].ontology is None -async def test_dataset_features_no_access(expdb_test: AsyncConnection) -> None: +async def test_dataset_features_no_access(expdb_session: AsyncSession) -> None: with pytest.raises(DatasetNoAccessError): - await get_dataset_features(dataset_id=130, user=None, expdb=expdb_test) + await get_dataset_features(dataset_id=130, user=None, expdb=expdb_session) @pytest.mark.parametrize("user", [ADMIN_USER, DATASET_130_OWNER]) -async def test_dataset_features_access_to_private(user: User, expdb_test: AsyncConnection) -> None: - features = await get_dataset_features(dataset_id=130, user=user, expdb=expdb_test) +async def test_dataset_features_access_to_private(user: User, expdb_session: AsyncSession) -> None: + features = await get_dataset_features(dataset_id=130, user=user, expdb=expdb_session) assert isinstance(features, list) -async def test_dataset_features_with_processing_error(expdb_test: AsyncConnection) -> None: +async def test_dataset_features_with_processing_error(expdb_session: AsyncSession) -> None: dataset_id = 55 with pytest.raises(DatasetProcessingError) as e: - await get_dataset_features(dataset_id=dataset_id, user=None, expdb=expdb_test) + await get_dataset_features(dataset_id=dataset_id, user=None, expdb=expdb_session) assert "No features found" in e.value.detail assert str(dataset_id) in e.value.detail -async def test_dataset_features_dataset_does_not_exist(expdb_test: AsyncConnection) -> None: +async def test_dataset_features_dataset_does_not_exist(expdb_session: AsyncSession) -> None: with pytest.raises(DatasetNotFoundError): - await get_dataset_features(dataset_id=1000, user=None, expdb=expdb_test) + await get_dataset_features(dataset_id=1000, user=None, expdb=expdb_session) # -- migration tests -- diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index 46e1aa9a..2c555f1f 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession async def test_get_dataset_via_api(py_api: httpx.AsyncClient) -> None: @@ -75,13 +75,13 @@ async def test_rfc9457_error_format(py_api: httpx.AsyncClient) -> None: @pytest.mark.mut async def test_dataset_no_500_with_multiple_processing_entries( py_api: httpx.AsyncClient, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: """Regression test for issue #145: multiple processing entries caused 500.""" - await expdb_test.execute( + await expdb_session.execute( text("INSERT INTO evaluation_engine(id, name, description) VALUES (99, 'test_engine', '')"), ) - await expdb_test.execute( + await expdb_session.execute( text( "INSERT INTO data_processed(did, evaluation_engine_id, user_id, processing_date) " "VALUES (1, 99, 2, '2020-01-01 00:00:00')", @@ -97,15 +97,15 @@ async def test_dataset_no_500_with_multiple_processing_entries( ) async def test_get_dataset_not_found( dataset_id: Identifier, - expdb_test: AsyncConnection, - user_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, ) -> None: with pytest.raises(DatasetNotFoundError) as exc_info: await get_dataset( dataset_id=dataset_id, user=None, - user_db=user_test, - expdb_db=expdb_test, + user_db=userdb_session, + expdb_db=expdb_session, ) assert exc_info.value.status_code == HTTPStatus.NOT_FOUND _dataset_get_not_found_code = 111 @@ -122,15 +122,15 @@ async def test_get_dataset_not_found( ) async def test_private_dataset_no_access( user: User | None, - expdb_test: AsyncConnection, - user_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, ) -> None: with pytest.raises(DatasetNoAccessError) as e: await get_dataset( dataset_id=130, user=user, - user_db=user_test, - expdb_db=expdb_test, + user_db=userdb_session, + expdb_db=expdb_session, ) assert e.value.status_code == HTTPStatus.FORBIDDEN assert e.value.uri == DatasetNoAccessError.uri @@ -142,13 +142,13 @@ async def test_private_dataset_no_access( "user", [DATASET_130_OWNER, ADMIN_USER, pytest.param(SOME_USER, marks=pytest.mark.xfail)] ) async def test_private_dataset_access( - user: User, expdb_test: AsyncConnection, user_test: AsyncConnection + user: User, expdb_session: AsyncSession, userdb_session: AsyncSession ) -> None: dataset = await get_dataset( dataset_id=130, user=user, - user_db=user_test, - expdb_db=expdb_test, + user_db=userdb_session, + expdb_db=expdb_session, ) assert isinstance(dataset, DatasetMetadata) diff --git a/tests/routers/datasets_list_datasets_test.py b/tests/routers/datasets_list_datasets_test.py index 0c307b2f..196adba0 100644 --- a/tests/routers/datasets_list_datasets_test.py +++ b/tests/routers/datasets_list_datasets_test.py @@ -17,7 +17,7 @@ from tests.users import ADMIN_USER, DATASET_130_OWNER, SOME_USER, ApiKey if TYPE_CHECKING: - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def test_list_route(py_api: httpx.AsyncClient) -> None: @@ -134,13 +134,13 @@ async def test_list_data_identical( ], ) async def test_list_filter_active( - status: DatasetStatusFilter, amount: int, expdb_test: AsyncConnection + status: DatasetStatusFilter, amount: int, expdb_session: AsyncSession ) -> None: result = await list_datasets( pagination=Pagination(limit=constants.NUMBER_OF_DATASETS), status=status, user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == amount @@ -155,13 +155,13 @@ async def test_list_filter_active( ], ) async def test_list_accounts_privacy( - user: User | None, amount: int, expdb_test: AsyncConnection + user: User | None, amount: int, expdb_session: AsyncSession ) -> None: result = await list_datasets( pagination=Pagination(limit=1000), status=DatasetStatusFilter.ALL, user=user, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == amount @@ -170,14 +170,14 @@ async def test_list_accounts_privacy( ("name", "count"), [("abalone", 1), ("iris", 2)], ) -async def test_list_data_name_present(name: str, count: int, expdb_test: AsyncConnection) -> None: +async def test_list_data_name_present(name: str, count: int, expdb_session: AsyncSession) -> None: # The second iris dataset is private, so we need an admin user. result = await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, data_name=name, user=ADMIN_USER, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == count assert all(dataset["name"] == name for dataset in result) @@ -187,21 +187,21 @@ async def test_list_data_name_present(name: str, count: int, expdb_test: AsyncCo "name", ["ir", "long_name_without_overlap"], ) -async def test_list_data_name_absent(name: str, expdb_test: AsyncConnection) -> None: +async def test_list_data_name_absent(name: str, expdb_session: AsyncSession) -> None: with pytest.raises(NoResultsError): await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, data_name=name, user=ADMIN_USER, - expdb_db=expdb_test, + expdb_db=expdb_session, ) @pytest.mark.parametrize("limit", [None, 5, 10, 200]) @pytest.mark.parametrize("offset", [None, 0, 5, 129, 140]) async def test_list_pagination( - limit: int | None, offset: int | None, expdb_test: AsyncConnection + limit: int | None, offset: int | None, expdb_session: AsyncSession ) -> None: # dataset ids are contiguous until 131, then there are 161, 162, and 163. extra_datasets = [161, 162, 163] @@ -222,7 +222,7 @@ async def test_list_pagination( pagination=pagination, status=DatasetStatusFilter.ALL, user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, ) except NoResultsError: expect_empty_offset = 140 @@ -236,19 +236,19 @@ async def test_list_pagination( ("version", "count"), [(1, 100), (2, 7), (5, 1)], ) -async def test_list_data_version(version: int, count: int, expdb_test: AsyncConnection) -> None: +async def test_list_data_version(version: int, count: int, expdb_session: AsyncSession) -> None: result = await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, data_version=version, user=ADMIN_USER, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == count assert {dataset["version"] for dataset in result} == {version} -async def test_list_data_version_no_result(expdb_test: AsyncConnection) -> None: +async def test_list_data_version_no_result(expdb_session: AsyncSession) -> None: version_with_no_datasets = 42 with pytest.raises(NoResultsError): await list_datasets( @@ -256,7 +256,7 @@ async def test_list_data_version_no_result(expdb_test: AsyncConnection) -> None: status=DatasetStatusFilter.ALL, data_version=version_with_no_datasets, user=ADMIN_USER, - expdb_db=expdb_test, + expdb_db=expdb_session, ) @@ -266,7 +266,7 @@ async def test_list_data_version_no_result(expdb_test: AsyncConnection) -> None: [(1, 59), (2, 34), (16, 1)], ) async def test_list_uploader( - user_id: Identifier, count: int, user: User, expdb_test: AsyncConnection + user_id: Identifier, count: int, user: User, expdb_session: AsyncSession ) -> None: # The dataset of user 16 is private, so can not be retrieved by other users. owner_user_id = 16 @@ -276,7 +276,7 @@ async def test_list_uploader( status=DatasetStatusFilter.ALL, uploader=user_id, user=user, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == count except NoResultsError: @@ -288,13 +288,13 @@ async def test_list_uploader( "data_id", [[1], [1, 2, 3], [1, 2, 3, 3000], [1, 2, 3, 130]], ) -async def test_list_data_id(data_id: list[int], expdb_test: AsyncConnection) -> None: +async def test_list_data_id(data_id: list[int], expdb_session: AsyncSession) -> None: result = await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, data_id=data_id, user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, ) private_or_not_exist = {130, 3000} expected = set(data_id) - private_or_not_exist @@ -306,25 +306,25 @@ async def test_list_data_id(data_id: list[int], expdb_test: AsyncConnection) -> ("tag", "count"), [("study_14", 100), ("study_15", 1)], ) -async def test_list_data_tag(tag: str, count: int, expdb_test: AsyncConnection) -> None: +async def test_list_data_tag(tag: str, count: int, expdb_session: AsyncSession) -> None: result = await list_datasets( pagination=Pagination(limit=101), status=DatasetStatusFilter.ALL, tag=tag, user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, ) assert len(result) == count -async def test_list_data_tag_empty(expdb_test: AsyncConnection) -> None: +async def test_list_data_tag_empty(expdb_session: AsyncSession) -> None: with pytest.raises(NoResultsError): await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, tag="not-a-tag", user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, ) @@ -342,13 +342,13 @@ async def test_list_data_tag_empty(expdb_test: AsyncConnection) -> None: ], ) async def test_list_data_quality( - quality: str, range_: str, count: int, expdb_test: AsyncConnection + quality: str, range_: str, count: int, expdb_session: AsyncSession ) -> None: result = await list_datasets( pagination=Pagination(), status=DatasetStatusFilter.ALL, user=None, - expdb_db=expdb_test, + expdb_db=expdb_session, **{quality: range_}, # type: ignore[arg-type] ) assert len(result) == count diff --git a/tests/routers/datasets_status_test.py b/tests/routers/datasets_status_test.py index 4e9a3164..b3b8be1a 100644 --- a/tests/routers/datasets_status_test.py +++ b/tests/routers/datasets_status_test.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def test_update_status_via_api(py_api: httpx.AsyncClient) -> None: @@ -29,55 +29,55 @@ async def test_update_status_via_api(py_api: httpx.AsyncClient) -> None: @pytest.mark.mut @pytest.mark.parametrize("dataset_id", [3, 4]) async def test_dataset_status_update_active_to_deactivated( - dataset_id: Identifier, expdb_test: AsyncConnection + dataset_id: Identifier, expdb_session: AsyncSession ) -> None: result = await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.DEACTIVATED, user=ADMIN_USER, - expdb=expdb_test, + expdb=expdb_session, ) assert result == {"dataset_id": dataset_id, "status": DatasetStatus.DEACTIVATED} @pytest.mark.mut async def test_dataset_status_update_in_preparation_to_active( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: dataset_id = next(iter(constants.IN_PREPARATION_ID)) result = await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.ACTIVE, user=ADMIN_USER, - expdb=expdb_test, + expdb=expdb_session, ) assert result == {"dataset_id": dataset_id, "status": DatasetStatus.ACTIVE} @pytest.mark.mut async def test_dataset_status_update_in_preparation_to_deactivated( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: dataset_id = next(iter(constants.IN_PREPARATION_ID)) result = await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.DEACTIVATED, user=ADMIN_USER, - expdb=expdb_test, + expdb=expdb_session, ) assert result == {"dataset_id": dataset_id, "status": DatasetStatus.DEACTIVATED} @pytest.mark.mut async def test_dataset_status_update_deactivated_to_active( - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: dataset_id = next(iter(constants.DEACTIVATED_DATASETS)) result = await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.ACTIVE, user=ADMIN_USER, - expdb=expdb_test, + expdb=expdb_session, ) assert result == {"dataset_id": dataset_id, "status": DatasetStatus.ACTIVE} @@ -85,26 +85,26 @@ async def test_dataset_status_update_deactivated_to_active( @pytest.mark.parametrize("dataset_id", [1, 33, 131]) async def test_dataset_status_non_admin_cannot_activate( dataset_id: Identifier, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: with pytest.raises(DatasetAdminOnlyError): await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.ACTIVE, user=SOME_USER, - expdb=expdb_test, + expdb=expdb_session, ) @pytest.mark.parametrize("dataset_id", [1, 2]) async def test_dataset_status_non_owner_cannot_deactivate( dataset_id: Identifier, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: with pytest.raises(DatasetNotOwnedError): await update_dataset_status( dataset_id=dataset_id, status=DatasetStatus.DEACTIVATED, user=SOME_USER, - expdb=expdb_test, + expdb=expdb_session, ) diff --git a/tests/routers/flows_exists_test.py b/tests/routers/flows_exists_test.py index e86ea414..ba0fd627 100644 --- a/tests/routers/flows_exists_test.py +++ b/tests/routers/flows_exists_test.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: import httpx from pytest_mock import MockerFixture - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def test_flow_exists(flow: Flow, py_api: httpx.AsyncClient) -> None: @@ -43,14 +43,14 @@ async def test_flow_exists_not_exists(py_api: httpx.AsyncClient) -> None: async def test_flow_exists_calls_db_correctly( name: str, external_version: str, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, mocker: MockerFixture, ) -> None: mocked_db = mocker.patch( "database.flows.get_by_name", new_callable=mocker.AsyncMock, ) - await flow_exists(name, external_version, expdb_test) + await flow_exists(name, external_version, expdb_session) mocked_db.assert_called_once_with( name=name, external_version=external_version, @@ -65,7 +65,7 @@ async def test_flow_exists_calls_db_correctly( async def test_flow_exists_processes_found( flow_id: Identifier, mocker: MockerFixture, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: fake_flow = mocker.MagicMock(id=flow_id) mocker.patch( @@ -73,16 +73,16 @@ async def test_flow_exists_processes_found( new_callable=mocker.AsyncMock, return_value=fake_flow, ) - response = await flow_exists("name", "external_version", expdb_test) + response = await flow_exists("name", "external_version", expdb_session) assert response == {"flow_id": fake_flow.id} async def test_flow_exists_handles_flow_not_found( - mocker: MockerFixture, expdb_test: AsyncConnection + mocker: MockerFixture, expdb_session: AsyncSession ) -> None: mocker.patch("database.flows.get_by_name", return_value=None) with pytest.raises(FlowNotFoundError) as error: - await flow_exists("foo", "bar", expdb_test) + await flow_exists("foo", "bar", expdb_session) assert error.value.status_code == HTTPStatus.NOT_FOUND assert error.value.uri == FlowNotFoundError.uri diff --git a/tests/routers/qualities_list_test.py b/tests/routers/qualities_list_test.py index 84cb94b2..ba02986e 100644 --- a/tests/routers/qualities_list_test.py +++ b/tests/routers/qualities_list_test.py @@ -7,27 +7,27 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession -async def _remove_quality_from_database(quality_name: str, expdb_test: AsyncConnection) -> None: - await expdb_test.execute( +async def _remove_quality_from_database(quality_name: str, expdb_session: AsyncSession) -> None: + await expdb_session.execute( text( """ DELETE FROM data_quality WHERE `quality`=:deleted_quality """, ), - parameters={"deleted_quality": quality_name}, + params={"deleted_quality": quality_name}, ) - await expdb_test.execute( + await expdb_session.execute( text( """ DELETE FROM quality WHERE `name`=:deleted_quality """, ), - parameters={"deleted_quality": quality_name}, + params={"deleted_quality": quality_name}, ) @@ -44,7 +44,7 @@ async def test_list_qualities_identical( @pytest.mark.mut -async def test_list_qualities(py_api: httpx.AsyncClient, expdb_test: AsyncConnection) -> None: +async def test_list_qualities(py_api: httpx.AsyncClient, expdb_session: AsyncSession) -> None: response = await py_api.get("/datasets/qualities/list") assert response.status_code == HTTPStatus.OK expected = { @@ -163,7 +163,7 @@ async def test_list_qualities(py_api: httpx.AsyncClient, expdb_test: AsyncConnec assert response.json() == expected deleted = expected["data_qualities_list"]["quality"].pop() - await _remove_quality_from_database(quality_name=deleted, expdb_test=expdb_test) + await _remove_quality_from_database(quality_name=deleted, expdb_session=expdb_session) response = await py_api.get("/datasets/qualities/list") assert response.status_code == HTTPStatus.OK diff --git a/tests/routers/setups_tag_test.py b/tests/routers/setups_tag_test.py index 49bad891..ec267008 100644 --- a/tests/routers/setups_tag_test.py +++ b/tests/routers/setups_tag_test.py @@ -26,7 +26,7 @@ async def test_setup_tag_missing_auth(py_api: httpx.AsyncClient) -> None: @pytest.mark.mut async def test_setup_tag_api_success( - py_api: httpx.AsyncClient, expdb_test: AsyncConnection + py_api: httpx.AsyncClient, expdb_session: AsyncSession ) -> None: tag = "setup_tag_via_http" response = await py_api.post( @@ -38,9 +38,9 @@ async def test_setup_tag_api_success( expected = {"setup_tag": {"id": "1", "tag": ["setup_tag_via_http"]}} assert response.json() == expected - rows = await expdb_test.execute( + rows = await expdb_session.execute( text("SELECT * FROM setup_tag WHERE id = 1 AND tag = :tag"), - parameters={"tag": tag}, + params={"tag": tag}, ) assert len(rows.all()) == 1 @@ -101,7 +101,7 @@ async def test_setup_tag_response_is_identical_when_tag_doesnt_exist( # noqa: P other_tags: list[str], py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, temporary_tags: Callable[..., AbstractAsyncContextManager[None]], ) -> None: setup_id = 1 @@ -115,11 +115,11 @@ async def test_setup_tag_response_is_identical_when_tag_doesnt_exist( # noqa: P data={"api_key": api_key, "tag": tag, "setup_id": setup_id}, ) - await expdb_test.execute( + await expdb_session.execute( text("DELETE FROM setup_tag WHERE `id`=:setup_id AND `tag`=:tag"), - parameters={"setup_id": setup_id, "tag": tag}, + params={"setup_id": setup_id, "tag": tag}, ) - await expdb_test.commit() + await expdb_session.commit() async with temporary_tags(table="setup_tag", tags=other_tags, identifier=setup_id): py_response = await py_api.post( diff --git a/tests/routers/setups_untag_test.py b/tests/routers/setups_untag_test.py index 1259f1e0..39dac022 100644 --- a/tests/routers/setups_untag_test.py +++ b/tests/routers/setups_untag_test.py @@ -26,12 +26,12 @@ async def test_setup_untag_missing_auth(py_api: httpx.AsyncClient) -> None: @pytest.mark.mut async def test_setup_untag_api_success( - py_api: httpx.AsyncClient, expdb_test: AsyncConnection + py_api: httpx.AsyncClient, expdb_session: AsyncSession ) -> None: tag = "setup_untag_via_http" - await expdb_test.execute( + await expdb_session.execute( text("INSERT INTO setup_tag (id, tag, uploader) VALUES (1, :tag, 2);"), - parameters={"tag": tag}, + params={"tag": tag}, ) response = await py_api.post( @@ -43,9 +43,9 @@ async def test_setup_untag_api_success( expected = {"setup_untag": {"id": "1", "tag": []}} assert response.json() == expected - rows = await expdb_test.execute( + rows = await expdb_session.execute( text("SELECT * FROM setup_tag WHERE id = 1 AND tag = :tag"), - parameters={"tag": tag}, + params={"tag": tag}, ) assert len(rows.all()) == 0 diff --git a/tests/routers/study_attach_test.py b/tests/routers/study_attach_test.py index 928c76ab..c1d56593 100644 --- a/tests/routers/study_attach_test.py +++ b/tests/routers/study_attach_test.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession async def _attach_tasks_to_study( @@ -19,13 +19,13 @@ async def _attach_tasks_to_study( task_ids: list[Identifier], api_key: str, py_api: httpx.AsyncClient, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> httpx.Response: # Adding requires the study to be in preparation, # but the current snapshot has no in-preparation studies. - await expdb_test.execute( + await expdb_session.execute( text("UPDATE study SET status = 'in_preparation' WHERE id = :study_id"), - parameters={"study_id": study_id}, + params={"study_id": study_id}, ) return await py_api.post( f"/studies/attach?api_key={api_key}", @@ -34,13 +34,13 @@ async def _attach_tasks_to_study( @pytest.mark.mut -async def test_attach_task_to_study(py_api: httpx.AsyncClient, expdb_test: AsyncConnection) -> None: +async def test_attach_task_to_study(py_api: httpx.AsyncClient, expdb_session: AsyncSession) -> None: response = await _attach_tasks_to_study( study_id=1, task_ids=[2, 3, 4], api_key=ApiKey.ADMIN, py_api=py_api, - expdb_test=expdb_test, + expdb_session=expdb_session, ) assert response.status_code == HTTPStatus.OK, response.content assert response.json() == {"study_id": 1, "main_entity_type": StudyType.TASK} @@ -48,15 +48,15 @@ async def test_attach_task_to_study(py_api: httpx.AsyncClient, expdb_test: Async @pytest.mark.mut async def test_attach_task_to_study_needs_owner( - py_api: httpx.AsyncClient, expdb_test: AsyncConnection + py_api: httpx.AsyncClient, expdb_session: AsyncSession ) -> None: - await expdb_test.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) + await expdb_session.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) response = await _attach_tasks_to_study( study_id=1, task_ids=[2, 3, 4], api_key=ApiKey.OWNER_USER, py_api=py_api, - expdb_test=expdb_test, + expdb_session=expdb_session, ) assert response.status_code == HTTPStatus.FORBIDDEN, response.content @@ -64,15 +64,15 @@ async def test_attach_task_to_study_needs_owner( @pytest.mark.mut async def test_attach_task_to_study_already_linked_raises( py_api: httpx.AsyncClient, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: - await expdb_test.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) + await expdb_session.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) response = await _attach_tasks_to_study( study_id=1, task_ids=[1, 3, 4], api_key=ApiKey.ADMIN, py_api=py_api, - expdb_test=expdb_test, + expdb_session=expdb_session, ) assert response.status_code == HTTPStatus.CONFLICT, response.content assert response.headers["content-type"] == "application/problem+json" @@ -84,15 +84,15 @@ async def test_attach_task_to_study_already_linked_raises( @pytest.mark.mut async def test_attach_task_to_study_but_task_not_exist_raises( py_api: httpx.AsyncClient, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, ) -> None: - await expdb_test.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) + await expdb_session.execute(text("UPDATE study SET status = 'in_preparation' WHERE id = 1")) response = await _attach_tasks_to_study( study_id=1, task_ids=[80123, 78914], api_key=ApiKey.ADMIN, py_api=py_api, - expdb_test=expdb_test, + expdb_session=expdb_session, ) assert response.status_code == HTTPStatus.CONFLICT assert response.headers["content-type"] == "application/problem+json" diff --git a/tests/routers/task_list_test.py b/tests/routers/task_list_test.py index bf52c1d9..b5eb37db 100644 --- a/tests/routers/task_list_test.py +++ b/tests/routers/task_list_test.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection + from sqlalchemy.ext.asyncio import AsyncSession async def test_list_tasks_default(py_api: httpx.AsyncClient) -> None: @@ -130,52 +130,52 @@ async def test_list_tasks_no_results_api_mapping(py_api: httpx.AsyncClient) -> N # ── Direct call tests: list_tasks ── -async def test_list_tasks_filter_type(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_filter_type(expdb_session: AsyncSession) -> None: """Filter by task_type_id returns only tasks of that type.""" - tasks = await list_tasks(pagination=Pagination(), task_type_id=1, expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(), task_type_id=1, expdb=expdb_session) assert len(tasks) > 0 assert all(t["task_type_id"] == 1 for t in tasks) -async def test_list_tasks_filter_tag(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_filter_tag(expdb_session: AsyncSession) -> None: """Filter by tag returns only tasks with that tag.""" - tasks = await list_tasks(pagination=Pagination(), tag="OpenML100", expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(), tag="OpenML100", expdb=expdb_session) assert len(tasks) > 0 assert all("OpenML100" in t["tag"] for t in tasks) @pytest.mark.parametrize("task_id", [1, 59, [1, 2, 3]]) async def test_list_tasks_filter_task_id( - task_id: Identifier | list[Identifier], expdb_test: AsyncConnection + task_id: Identifier | list[Identifier], expdb_session: AsyncSession ) -> None: """Filter by task_id returns only those tasks (regardless of status).""" ids = [task_id] if isinstance(task_id, int) else task_id tasks = await list_tasks( - pagination=Pagination(), task_id=ids, status=TaskStatusFilter.ALL, expdb=expdb_test + pagination=Pagination(), task_id=ids, status=TaskStatusFilter.ALL, expdb=expdb_session ) returned_ids = sorted(t["task_id"] for t in tasks) assert returned_ids == sorted(ids) -async def test_list_tasks_filter_data_id(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_filter_data_id(expdb_session: AsyncSession) -> None: """Filter by data_id returns only tasks that use that dataset.""" data_id = 10 - tasks = await list_tasks(pagination=Pagination(), data_id=[data_id], expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(), data_id=[data_id], expdb=expdb_session) assert len(tasks) > 0 assert all(t["did"] == data_id for t in tasks) -async def test_list_tasks_filter_data_name(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_filter_data_name(expdb_session: AsyncSession) -> None: """Filter by data_name returns only tasks whose dataset matches.""" - tasks = await list_tasks(pagination=Pagination(), data_name="mfeat-pixel", expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(), data_name="mfeat-pixel", expdb=expdb_session) assert len(tasks) > 0 assert all(t["name"] == "mfeat-pixel" for t in tasks) -async def test_list_tasks_filter_status_deactivated(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_filter_status_deactivated(expdb_session: AsyncSession) -> None: """Filter by status='deactivated' returns tasks with that status.""" tasks = await list_tasks( - pagination=Pagination(), status=TaskStatusFilter.DEACTIVATED, expdb=expdb_test + pagination=Pagination(), status=TaskStatusFilter.DEACTIVATED, expdb=expdb_session ) assert len(tasks) > 0 assert all(t["status"] == "deactivated" for t in tasks) @@ -185,23 +185,23 @@ async def test_list_tasks_filter_status_deactivated(expdb_test: AsyncConnection) ("limit", "offset"), [(5, 0), (10, 0), (5, 5)], ) -async def test_list_tasks_pagination(limit: int, offset: int, expdb_test: AsyncConnection) -> None: +async def test_list_tasks_pagination(limit: int, offset: int, expdb_session: AsyncSession) -> None: """Pagination limit and offset are respected.""" - tasks = await list_tasks(pagination=Pagination(limit=limit, offset=offset), expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(limit=limit, offset=offset), expdb=expdb_session) assert len(tasks) <= limit # Precise verification: compare IDs against a corresponding slice from an offset=0 baseline baseline = await list_tasks( - pagination=Pagination(limit=limit + offset, offset=0), expdb=expdb_test + pagination=Pagination(limit=limit + offset, offset=0), expdb=expdb_session ) expected_ids = [t["task_id"] for t in baseline][offset : offset + limit] assert [t["task_id"] for t in tasks] == expected_ids -async def test_list_tasks_pagination_order_stable(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_pagination_order_stable(expdb_session: AsyncSession) -> None: """Results are ordered by task_id — consecutive pages are in ascending order.""" - tasks1 = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_test) - tasks2 = await list_tasks(pagination=Pagination(limit=5, offset=5), expdb=expdb_test) + tasks1 = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_session) + tasks2 = await list_tasks(pagination=Pagination(limit=5, offset=5), expdb=expdb_session) ids1 = [t["task_id"] for t in tasks1] ids2 = [t["task_id"] for t in tasks2] assert ids1 == sorted(ids1) @@ -210,13 +210,13 @@ async def test_list_tasks_pagination_order_stable(expdb_test: AsyncConnection) - assert max(ids1) < min(ids2) -async def test_list_tasks_number_instances_range(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_number_instances_range(expdb_session: AsyncSession) -> None: """number_instances range filter returns tasks whose dataset matches.""" min_instances, max_instances = 100, 1000 tasks = await list_tasks( pagination=Pagination(), number_instances=f"{min_instances}..{max_instances}", - expdb=expdb_test, + expdb=expdb_session, ) assert len(tasks) > 0 for task in tasks: @@ -225,19 +225,19 @@ async def test_list_tasks_number_instances_range(expdb_test: AsyncConnection) -> assert min_instances <= float(qualities["NumberOfInstances"]) <= max_instances -async def test_list_tasks_inputs_are_basic_subset(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_inputs_are_basic_subset(expdb_session: AsyncSession) -> None: """Input entries only contain the expected basic input names.""" basic_inputs = {"source_data", "target_feature", "estimation_procedure", "evaluation_measures"} - tasks = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_session) assert any(task["input"] for task in tasks), "Expected at least one task to have inputs" for task in tasks: for inp in task["input"]: assert inp["name"] in basic_inputs -async def test_list_tasks_quality_values_are_strings(expdb_test: AsyncConnection) -> None: +async def test_list_tasks_quality_values_are_strings(expdb_session: AsyncSession) -> None: """Quality values must be strings (to match PHP API behaviour).""" - tasks = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_test) + tasks = await list_tasks(pagination=Pagination(limit=5, offset=0), expdb=expdb_session) assert any(task["quality"] for task in tasks), "Expected at least one task to have qualities" qualities = [quality for task in tasks for quality in task["quality"]] assert all(isinstance(quality["value"], str) for quality in qualities) @@ -252,10 +252,10 @@ async def test_list_tasks_quality_values_are_strings(expdb_test: AsyncConnection ], ids=["bad_tag", "bad_task_id", "bad_data_name"], ) -async def test_list_tasks_no_results(payload: dict[str, Any], expdb_test: AsyncConnection) -> None: +async def test_list_tasks_no_results(payload: dict[str, Any], expdb_session: AsyncSession) -> None: """Filters matching nothing return 404 NoResultsError.""" with pytest.raises(NoResultsError): - await list_tasks(pagination=Pagination(), expdb=expdb_test, **payload) + await list_tasks(pagination=Pagination(), expdb=expdb_session, **payload) _TASK_LIST_NO_RESULTS_CODE = "482" diff --git a/tests/routers/users_delete_test.py b/tests/routers/users_delete_test.py index 9de53ddb..9f049014 100644 --- a/tests/routers/users_delete_test.py +++ b/tests/routers/users_delete_test.py @@ -9,7 +9,7 @@ import pytest_mock # noqa: TC002 used at runtime by pytest fixtures from sqlalchemy import text from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncConnection # noqa: TC002 used at runtime by pytest fixtures +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession # noqa: TC002 used at runtime by pytest fixtures from core.errors import AccountHasResourcesError, ForbiddenError, UserNotFoundError from core.types import Identifier @@ -32,13 +32,13 @@ class DisposableUser(NamedTuple): @pytest.fixture -async def disposable_user(user_test: AsyncConnection) -> DisposableUser: +async def disposable_user(userdb_session: AsyncSession) -> DisposableUser: api_key = uuid.uuid4().hex suffix = uuid.uuid4().hex[:10] username = f"tmp_user_{suffix}" email = f"{suffix}@openml-delete.test" - await user_test.execute( + await userdb_session.execute( text( """ INSERT INTO users ( @@ -50,13 +50,13 @@ async def disposable_user(user_test: AsyncConnection) -> DisposableUser: ) """, ), - parameters={"username": username, "email": email, "api_key": api_key}, + params={"username": username, "email": email, "api_key": api_key}, ) - uid_row = await user_test.execute(text("SELECT LAST_INSERT_ID() AS id")) + uid_row = await userdb_session.execute(text("SELECT LAST_INSERT_ID() AS id")) (new_id,) = uid_row.one() - await user_test.execute( + await userdb_session.execute( text("INSERT INTO users_groups (user_id, group_id) VALUES (:uid, :gid)"), - parameters={"uid": new_id, "gid": UserGroup.READ_WRITE.value}, + params={"uid": new_id, "gid": UserGroup.READ_WRITE.value}, ) return DisposableUser(user_id=new_id, api_key=api_key) # No explicit teardown: the ``user_test`` fixture rolls back at the end @@ -66,7 +66,7 @@ async def disposable_user(user_test: AsyncConnection) -> DisposableUser: @pytest.mark.mut async def test_delete_user_api_success_self_delete( py_api: httpx.AsyncClient, - user_test: AsyncConnection, + userdb_session: AsyncSession, disposable_user: DisposableUser, mocker: pytest_mock.MockerFixture, ) -> None: @@ -79,9 +79,9 @@ async def test_delete_user_api_success_self_delete( assert response.status_code == HTTPStatus.NO_CONTENT assert response.content == b"" - exists = await user_test.execute( + exists = await userdb_session.execute( text("SELECT 1 FROM users WHERE id = :id LIMIT 1"), - parameters={"id": disposable_user.user_id}, + params={"id": disposable_user.user_id}, ) assert exists.one_or_none() is None @@ -94,7 +94,7 @@ async def test_delete_user_api_success_self_delete( @pytest.mark.mut async def test_delete_user_api_success_admin_deletes_disposable_user( py_api: httpx.AsyncClient, - user_test: AsyncConnection, + userdb_session: AsyncSession, disposable_user: DisposableUser, ) -> None: response = await py_api.delete( @@ -104,9 +104,9 @@ async def test_delete_user_api_success_admin_deletes_disposable_user( assert response.status_code == HTTPStatus.NO_CONTENT assert response.content == b"" - exists = await user_test.execute( + exists = await userdb_session.execute( text("SELECT 1 FROM users WHERE id = :id LIMIT 1"), - parameters={"id": disposable_user.user_id}, + params={"id": disposable_user.user_id}, ) assert exists.one_or_none() is None @@ -115,23 +115,23 @@ async def test_delete_user_api_success_admin_deletes_disposable_user( async def test_delete_user_direct_not_found( - user_test: AsyncConnection, - expdb_test: AsyncConnection, + userdb_session: AsyncSession, + expdb_session: AsyncSession, ) -> None: with pytest.raises(UserNotFoundError, match=r"User 888888888 not found\.") as exc_info: await delete_user_account( user_id=888888888, current_user=ADMIN_USER, - expdb=expdb_test, - userdb=user_test, + expdb=expdb_session, + userdb=userdb_session, ) assert exc_info.value.status_code == HTTPStatus.NOT_FOUND assert exc_info.value.uri == UserNotFoundError.uri async def test_delete_user_direct_forbidden( - user_test: AsyncConnection, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, ) -> None: with pytest.raises( ForbiddenError, match=r"You may only delete your own user account\." @@ -139,44 +139,44 @@ async def test_delete_user_direct_forbidden( await delete_user_account( user_id=ADMIN_USER.user_id, current_user=SOME_USER, - expdb=expdb_test, - userdb=user_test, + expdb=expdb_session, + userdb=userdb_session, ) assert exc_info.value.status_code == HTTPStatus.FORBIDDEN assert exc_info.value.uri == ForbiddenError.uri - admin_row = await user_test.execute( + admin_row = await userdb_session.execute( text("SELECT 1 FROM users WHERE id = :id LIMIT 1"), - parameters={"id": ADMIN_USER.user_id}, + params={"id": ADMIN_USER.user_id}, ) assert admin_row.one_or_none() is not None async def test_delete_user_direct_conflict_has_resources( - user_test: AsyncConnection, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, ) -> None: with pytest.raises(AccountHasResourcesError, match="Cannot delete this account") as exc_info: await delete_user_account( user_id=OWNER_USER.user_id, current_user=ADMIN_USER, - expdb=expdb_test, - userdb=user_test, + expdb=expdb_session, + userdb=userdb_session, ) assert exc_info.value.status_code == HTTPStatus.CONFLICT assert exc_info.value.uri == AccountHasResourcesError.uri - owner_row = await user_test.execute( + owner_row = await userdb_session.execute( text("SELECT 1 FROM users WHERE id = :id LIMIT 1"), - parameters={"id": OWNER_USER.user_id}, + params={"id": OWNER_USER.user_id}, ) assert owner_row.one_or_none() is not None @pytest.mark.mut async def test_delete_user_integrity_error_logs_and_raises_conflict( - user_test: AsyncConnection, - expdb_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, disposable_user: DisposableUser, mocker: pytest_mock.MockerFixture, ) -> None: @@ -192,8 +192,8 @@ async def test_delete_user_integrity_error_logs_and_raises_conflict( await delete_user_account( user_id=disposable_user.user_id, current_user=ADMIN_USER, - expdb=expdb_test, - userdb=user_test, + expdb=expdb_session, + userdb=userdb_session, ) assert exc_info.value.status_code == HTTPStatus.CONFLICT From 3bcb9b3920ba9cbc0faac62b46eaed06124a0c9e Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 16:24:24 +0200 Subject: [PATCH 09/14] Do not use AsyncSession concurrently together, it's not supported --- src/routers/datasets.py | 25 +++++++--------- src/routers/flows.py | 9 +++--- src/routers/runs.py | 33 +++++++------------- src/routers/tasks.py | 41 +++++++++++-------------- tests/conftest.py | 48 ++++++++++-------------------- tests/routers/datasets_get_test.py | 12 ++++---- 6 files changed, 64 insertions(+), 104 deletions(-) diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 530234cf..68b48831 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -19,6 +19,7 @@ from sqlalchemy import bindparam, text import database.datasets +from database.models import tags import database.qualities from config import get_config from core.access import user_has_access @@ -394,10 +395,8 @@ async def get_dataset_features( """Return metadata for each feature (column) in the dataset.""" assert expdb is not None # noqa: S101 await _get_dataset_raise_otherwise(dataset_id, user, expdb) - features, ontologies = await asyncio.gather( - database.datasets.get_features(dataset_id, expdb), - database.datasets.get_feature_ontologies(dataset_id, expdb), - ) + features = await database.datasets.get_features(dataset_id, expdb) + ontologies = await database.datasets.get_feature_ontologies(dataset_id, expdb) for feature in features: feature.ontology = ontologies.get(feature.index) @@ -487,27 +486,25 @@ async def update_dataset_status( ) async def get_dataset( dataset_id: Identifier, - user_db: Annotated[AsyncSession, Depends(userdb_session)], - expdb_db: Annotated[AsyncSession, Depends(expdb_session)], + userdb_session: Annotated[AsyncSession, Depends(userdb_session)], + expdb_session: Annotated[AsyncSession, Depends(expdb_session)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> DatasetMetadata: """Get the user-provided metadata for a dataset.""" - dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb_db) + dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb_session) if not ( dataset_file := await database.datasets.get_file( file_id=dataset.file_id, - session=user_db, + session=userdb_session, ) ): msg = f"No data file found for dataset {dataset_id}." raise DatasetNoDataFileError(msg) - tags, description, processing_result, status = await asyncio.gather( - database.datasets.get_tags_for(dataset_id, expdb_db), - database.datasets.get_description(dataset_id, expdb_db), - _get_processing_information(dataset_id, expdb_db), - database.datasets.get_status(dataset_id, expdb_db), - ) + tags = await database.datasets.get_tags_for(dataset_id, expdb_session) + description = await database.datasets.get_description(dataset_id, expdb_session) + processing_result = await _get_processing_information(dataset_id, expdb_session) + status = await database.datasets.get_status(dataset_id, expdb_session) description_ = "" if description: diff --git a/src/routers/flows.py b/src/routers/flows.py index 2e3bd596..47a2418e 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -9,6 +9,7 @@ from core.conversions import str_to_num from core.errors import FlowNotFoundError from core.types import Identifier +from database.models import tags from routers.dependencies import expdb_session from routers.schemas.flows import Flow, Parameter, Subflow @@ -47,11 +48,9 @@ async def get_flow( msg = f"Flow with id {flow_id} not found." raise FlowNotFoundError(msg) - parameter_rows, tags, subflow_rows = await asyncio.gather( - database.flows.get_parameters(flow_id, expdb), - database.flows.get_tags(flow_id, expdb), - database.flows.get_subflows(flow_id, expdb), - ) + parameter_rows = await database.flows.get_parameters(flow_id, expdb) + tags = await database.flows.get_tags(flow_id, expdb) + subflow_rows = await database.flows.get_subflows(flow_id, expdb) parameters = [ Parameter( name=parameter.name, diff --git a/src/routers/runs.py b/src/routers/runs.py index e7904f75..b030bea4 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -86,30 +86,17 @@ async def _load_run_context( userdb: AsyncSession, engine_ids: list[int], ) -> RunContext: - ( - uploader_user, - tags, - input_data_rows, - output_file_rows, - evaluation_rows, - task_type, - task_evaluation_measure, - setup, - parameter_rows, - ) = cast( - "tuple[Any, list[str], list[UntypedRow], list[UntypedRow], list[UntypedRow], str | None, str | None, Setup | None, list[UntypedRow]]", # noqa: E501 - await asyncio.gather( - database.users.get_user(user_id=run.uploader, session=userdb), - database.runs.get_tags(run.rid, expdb), - database.runs.get_input_data(run.rid, expdb), - database.runs.get_output_files(run.rid, expdb), - database.runs.get_evaluations(run.rid, expdb, evaluation_engine_ids=engine_ids), - database.tasks.get_task_type_name(run.task_id, expdb), - database.tasks.get_task_evaluation_measure(run.task_id, expdb), - database.setups.get(run.setup, expdb), - database.setups.get_parameters(run.setup, expdb), - ), + uploader_user = await database.users.get_user(user_id=run.uploader, session=userdb) + tags = await database.runs.get_tags(run.rid, expdb) + input_data_rows = await database.runs.get_input_data(run.rid, expdb) + output_file_rows = await database.runs.get_output_files(run.rid, expdb) + evaluation_rows = await database.runs.get_evaluations( + run.rid, expdb, evaluation_engine_ids=engine_ids ) + task_type = await database.tasks.get_task_type_name(run.task_id, expdb) + task_evaluation_measure = await database.tasks.get_task_evaluation_measure(run.task_id, expdb) + setup = await database.setups.get(run.setup, expdb) + parameter_rows = await database.setups.get_parameters(run.setup, expdb) return RunContext( uploader_name=uploader_user.full_name if uploader_user else None, tags=tags, diff --git a/src/routers/tasks.py b/src/routers/tasks.py index cb91c997..27043998 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -152,8 +152,7 @@ async def _fill_json_template( # noqa: C901 } if isinstance(template, list): return [ - await _fill_json_template(v, task, task_inputs, fetched_data, session) - for v in template + await _fill_json_template(v, task, task_inputs, fetched_data, session) for v in template ] if not isinstance(template, str): msg = f"Unexpected type for `template`: {template=}, {type(template)=}" @@ -414,19 +413,17 @@ async def list_tasks( # noqa: PLR0913, PLR0912, C901, PLR0915 """, ).bindparams(bindparam("task_ids", expanding=True)) - inputs_result, qualities_result, tags_result = await asyncio.gather( - expdb.execute( - inputs_query, - params={"task_ids": task_ids, "basic_inputs": BASIC_TASK_INPUTS}, - ), - expdb.execute( - qualities_query, - params={"dataset_ids": dataset_ids, "quality_names": QUALITIES_TO_SHOW}, - ), - expdb.execute( - tags_query, - params={"task_ids": task_ids}, - ), + inputs_result = await expdb.execute( + inputs_query, + params={"task_ids": task_ids, "basic_inputs": BASIC_TASK_INPUTS}, + ) + qualities_result = await expdb.execute( + qualities_query, + params={"dataset_ids": dataset_ids, "quality_names": QUALITIES_TO_SHOW}, + ) + tags_result = await expdb.execute( + tags_query, + params={"task_ids": task_ids}, ) for row in inputs_result.all(): @@ -463,11 +460,9 @@ async def get_task( msg = f"Task {task_id} has task type {task.ttid}, but task type {task.ttid} is not found." raise InternalError(msg) - task_input_rows, ttios, tags = await asyncio.gather( - database.tasks.get_input_for_task(task_id, expdb), - database.tasks.get_task_type_inout_with_template(task_type.ttid, expdb), - database.tasks.get_tags(task_id, expdb), - ) + task_input_rows = await database.tasks.get_input_for_task(task_id, expdb) + ttios = await database.tasks.get_task_type_inout_with_template(task_type.ttid, expdb) + tags = await database.tasks.get_tags(task_id, expdb) task_inputs = { row.input: int(row.value) if row.value.isdigit() else row.value for row in task_input_rows } @@ -475,9 +470,9 @@ async def get_task( input_templates = [ (name, template) for name, io, required, template in templates if io == "input" ] - filled_templates = await asyncio.gather( - *[fill_template(template, task, task_inputs, expdb) for name, template in input_templates], - ) + filled_templates = [] + for name, template in input_templates: + filled_templates.append(await fill_template(template, task, task_inputs, expdb)) inputs = [ filled | {"name": name} for (name, _), filled in zip(input_templates, filled_templates, strict=True) diff --git a/tests/conftest.py b/tests/conftest.py index dbee65ba..08c69deb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,10 +38,9 @@ @contextlib.asynccontextmanager -async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncConnection]: - async with engine.connect() as connection: - transaction = await connection.begin() - yield connection +async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncSession]: + async with AsyncSession(engine) as session, session.begin() as transaction: + yield session if transaction.is_active: await transaction.rollback() @@ -70,32 +69,15 @@ async def temporary_records( @pytest.fixture -async def expdb_test() -> AsyncIterator[AsyncConnection]: - async with automatic_rollback(expdb_database()) as connection: - yield connection - - -@pytest.fixture -async def expdb_session(expdb_test: AsyncConnection) -> AsyncIterator[AsyncSession]: - # It is possible that this session `commits`, does the connection - # rollback then still take effect? Probably not. - async with AsyncSession(expdb_test) as session: +async def expdb_session() -> AsyncIterator[AsyncSession]: + async with automatic_rollback(expdb_database()) as session: yield session - # Do we here again need to do some check on whether there is an active transation? - await session.rollback() @pytest.fixture -async def user_test() -> AsyncIterator[AsyncConnection]: - async with automatic_rollback(user_database()) as connection: - yield connection - - -@pytest.fixture -async def userdb_session(user_test: AsyncConnection) -> AsyncIterator[AsyncSession]: - async with AsyncSession(user_test) as session: +async def userdb_session() -> AsyncIterator[AsyncSession]: + async with automatic_rollback(user_database()) as session: yield session - await session.rollback() # The PHP API fixture can be session scoped since they do not need access to @@ -124,8 +106,8 @@ async def app() -> AsyncIterator[FastAPI]: @pytest.fixture async def py_api( - expdb_test: AsyncConnection, - user_test: AsyncConnection, + expdb_session: AsyncSession, + userdb_session: AsyncSession, app: FastAPI, ) -> AsyncIterator[httpx.AsyncClient]: """Create test client which automatically rolls back database updates on teardown.""" @@ -133,17 +115,17 @@ async def py_api( # automatic rollbacks, but also lets a test author write to a database # transaction that is shared with the app. That is, it enables: # - # def my_test(expdb_test, py_api): - # expdb_test.execute(...) # write some data # noqa: ERA001 + # def my_test(expdb_session, py_api): + # expdb_session.execute(...) # write some data # noqa: ERA001 # py_api.get(...) # read that data # noqa: ERA001 async def override_expdb() -> AsyncIterator[AsyncSession]: - async with AsyncSession(expdb_test) as session: - yield session + async with expdb_session.begin_nested(): + yield expdb_session async def override_userdb() -> AsyncIterator[AsyncSession]: - async with AsyncSession(user_test) as session: - yield session + async with userdb_session.begin_nested(): + yield userdb_session app.dependency_overrides[expdb_session_dep] = override_expdb app.dependency_overrides[userdb_session_dep] = override_userdb diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index 2c555f1f..4a3317f2 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -104,8 +104,8 @@ async def test_get_dataset_not_found( await get_dataset( dataset_id=dataset_id, user=None, - user_db=userdb_session, - expdb_db=expdb_session, + userdb_session=userdb_session, + expdb_session=expdb_session, ) assert exc_info.value.status_code == HTTPStatus.NOT_FOUND _dataset_get_not_found_code = 111 @@ -129,8 +129,8 @@ async def test_private_dataset_no_access( await get_dataset( dataset_id=130, user=user, - user_db=userdb_session, - expdb_db=expdb_session, + userdb_session=userdb_session, + expdb_session=expdb_session, ) assert e.value.status_code == HTTPStatus.FORBIDDEN assert e.value.uri == DatasetNoAccessError.uri @@ -147,8 +147,8 @@ async def test_private_dataset_access( dataset = await get_dataset( dataset_id=130, user=user, - user_db=userdb_session, - expdb_db=expdb_session, + userdb_session=userdb_session, + expdb_session=expdb_session, ) assert isinstance(dataset, DatasetMetadata) From c33aafd4aed52ee1489155d6293d7667f707749b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 17:46:27 +0200 Subject: [PATCH 10/14] Troubleshooting the session/connection lifecycle --- tests/conftest.py | 79 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 08c69deb..354eaea9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,8 @@ import httpx import pytest from _pytest.config import Config # noqa: TC002 used during collection by Pytest -from _pytest.nodes import Item # noqa: TC002 used during collection by Pytest +from _pytest.nodes import Item +from sqlalchemy.orm import Session # noqa: TC002 used during collection by Pytest from asgi_lifespan import LifespanManager from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession @@ -25,6 +26,8 @@ from database.engine import expdb_database, user_database from main import create_api from routers.dependencies import ( + expdb_connection as expdb_connection_dep, + userdb_connection as userdb_connection_dep, expdb_session as expdb_session_dep, userdb_session as userdb_session_dep, ) @@ -38,16 +41,16 @@ @contextlib.asynccontextmanager -async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncSession]: - async with AsyncSession(engine) as session, session.begin() as transaction: - yield session +async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncConnection]: + async with engine.connect() as connection, connection.begin() as transaction: + yield connection if transaction.is_active: await transaction.rollback() @contextlib.asynccontextmanager async def temporary_records( - session: AsyncSession, + connection: AsyncConnection, insert_queries: Iterable[tuple[str, dict[str, Any] | None]], delete_queries: Iterable[tuple[str, dict[str, Any] | None]], *, @@ -55,28 +58,44 @@ async def temporary_records( ) -> AsyncIterator[None]: """Execute insert queries on enter and their corresponding delete queries on exit.""" for query, parameters in insert_queries: - await session.execute(text(query), params=parameters) + await connection.execute(text(query), parameters=parameters) if persist: - await session.commit() + await connection.commit() try: yield finally: for query, parameters in delete_queries: - await session.execute(text(query), params=parameters) + await connection.execute(text(query), parameters=parameters) if persist: - await session.commit() + await connection.commit() + + +@pytest.fixture +async def expdb_connection() -> AsyncIterator[AsyncConnection]: + async with automatic_rollback(expdb_database()) as connection: + yield connection + + +@pytest.fixture +async def userdb_connection() -> AsyncIterator[AsyncConnection]: + async with automatic_rollback(user_database()) as connection: + yield connection @pytest.fixture -async def expdb_session() -> AsyncIterator[AsyncSession]: - async with automatic_rollback(expdb_database()) as session: +async def expdb_session(expdb_connection: AsyncConnection) -> AsyncIterator[AsyncSession]: + async with AsyncSession( + bind=expdb_connection, join_transaction_mode="create_savepoint" + ) as session: yield session @pytest.fixture -async def userdb_session() -> AsyncIterator[AsyncSession]: - async with automatic_rollback(user_database()) as session: +async def userdb_session(userdb_connection: AsyncConnection) -> AsyncIterator[AsyncSession]: + async with AsyncSession( + bind=userdb_connection, join_transaction_mode="create_savepoint" + ) as session: yield session @@ -106,6 +125,8 @@ async def app() -> AsyncIterator[FastAPI]: @pytest.fixture async def py_api( + expdb_connection: AsyncConnection, + userdb_connection: AsyncConnection, expdb_session: AsyncSession, userdb_session: AsyncSession, app: FastAPI, @@ -119,16 +140,22 @@ async def py_api( # expdb_session.execute(...) # write some data # noqa: ERA001 # py_api.get(...) # read that data # noqa: ERA001 - async def override_expdb() -> AsyncIterator[AsyncSession]: - async with expdb_session.begin_nested(): - yield expdb_session + async def override_expdb_connection() -> AsyncIterator[AsyncConnection]: + yield expdb_connection + + async def override_userdb_connection() -> AsyncIterator[AsyncConnection]: + yield userdb_connection - async def override_userdb() -> AsyncIterator[AsyncSession]: - async with userdb_session.begin_nested(): - yield userdb_session + async def override_expdb_session() -> AsyncIterator[AsyncSession]: + yield expdb_session - app.dependency_overrides[expdb_session_dep] = override_expdb - app.dependency_overrides[userdb_session_dep] = override_userdb + async def override_userdb_session() -> AsyncIterator[AsyncSession]: + yield userdb_session + + app.dependency_overrides[expdb_connection_dep] = override_expdb_connection + app.dependency_overrides[userdb_connection_dep] = override_userdb_connection + app.dependency_overrides[expdb_session_dep] = override_expdb_session + app.dependency_overrides[userdb_session_dep] = override_userdb_session async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), @@ -137,8 +164,10 @@ async def override_userdb() -> AsyncIterator[AsyncSession]: ) as client: yield client - app.dependency_overrides[expdb_session_dep] = expdb_session - app.dependency_overrides[userdb_session_dep] = userdb_session + app.dependency_overrides[expdb_connection_dep] = expdb_connection_dep + app.dependency_overrides[userdb_connection_dep] = userdb_connection_dep + app.dependency_overrides[expdb_session_dep] = expdb_session_dep + app.dependency_overrides[userdb_session_dep] = userdb_session_dep @pytest.fixture @@ -282,7 +311,7 @@ async def persisted_flow(flow: Flow, expdb_session: AsyncSession) -> AsyncIterat @pytest.fixture def temporary_tags( - expdb_session: AsyncSession, + expdb_connection: AsyncConnection, ) -> Callable[..., contextlib.AbstractAsyncContextManager[None]]: @contextlib.asynccontextmanager async def _temporary_tags( @@ -311,7 +340,7 @@ async def _temporary_tags( for tag in tags ] async with temporary_records( - session=expdb_session, + connection=expdb_connection, insert_queries=insert_queries, delete_queries=delete_queries, persist=persist, From 532107b0a141742ff44225fbb594424bc180ce6d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 10 Jul 2026 18:29:26 +0200 Subject: [PATCH 11/14] Fix issue with prematurely closed connections --- docs/development/tests.md | 62 ++++++++++++++++++++++++++++++------- src/routers/dependencies.py | 2 +- src/routers/runs.py | 12 ++++--- tests/conftest.py | 30 +++++++++++------- 4 files changed, 78 insertions(+), 28 deletions(-) diff --git a/docs/development/tests.md b/docs/development/tests.md index 09ec893c..542c8160 100644 --- a/docs/development/tests.md +++ b/docs/development/tests.md @@ -71,18 +71,58 @@ Some guidelines and things to keep in mind when writing tests: - Try to keep tests small, so that they fail for one particular reason only. - Mark tests that update the database in anyway with the `mut` marker (`@pytest.mark.mut`). - If the test is excessively slow (>0.1 sec) and does not connect to PHP, use a `slow` marker. Tests that include PHP always require roundtrips through other services which makes them slow by default. PHP tests can be filtered out with the automatically generated "php_api" marker. - - Four common fixtures you might need when writing tests are: - - `py_api`: an async client for the Python based REST API - - `php_api`: an async client for the PHP based REST API - - `expdb_test`: an AsyncConnection to the "expdb" OpenML database. - - `user_test`: an AsyncConnection to the "openml" OpenML database. - - Above fixtures have considerable per-test overhead. Use them only when you need them. - When writing assertions the expected value (a constant, or a php response) should be on the right (`assert response == expected`). +### Fixtures +There are a number of fixtures in `conftest.py`, here is a quick rundown of the most notable ones: + + - `py_api` and `php_api`: an async client for the Python- and PHP-based REST APIs, respectively. The `py_api` client has its normal dependency injection for database connections patched to use the connection and session fixtures below. + - `expdb_connection` and `userdb_connection`: an AsyncConnection to the "expdb" OpenML database. This fixture is function-scoped and automatically starts a transaction which is rolled back as long as no `commit` is made. + - `expdb_session` and `userdb_session`: an AsyncSession that is bound to the respective connection. This means it also has automatic rollback. Any changes that need to be visible to the `py_api` fixture can be performed on this session (see below). + +The pseudocode below shows how you might combine these for a test of the new REST API, either as standalone or when compared to the PHP API: + +```python + +async def test_python(py_api: httpx.AsyncClient, expdb_session: AsyncSession) -> None: + await expdb_session.execute(text("INSERT INTO dataset ..."), params=...) # Insert dataset with id 42 + + response = await py_api.get("/datasets/42") # Since this call shares the session, it should retrieve this data + + assert ... + # after the test is done, the fixture clean up will ensure the change is not committed to the database, no extra code needed + +async def test_python_and_php(py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, expdb_connection: AsyncConnection) -> None: + await expdb_connection.execute(text("INSERT INTO dataset ..."), params=...) # Insert dataset with id 42 + await expdb_connection.commit() # We need to persist the data in the database, because the PHP REST API cannot see our transaction + + response = await php_api.get("/datasets/42") # The PHP REST API can see the dataset, because it exists in the database + response = await py_api.get("/datasets/42") # The Python REST API can see the dataset also + + # We need to clean up after ourselves, otherwise the test has side effects. + # This isn't a great pattern, prefer instead the use of context managers which will execute the delete statements even if unexpected exceptions occur. + await expdb_connection.execute(text("DELETE FROM dataset ..."), params=...) + await expdb_connection.commit() + +``` + +???- "Why not always use the `*_connection`?" + + As this implementation will make more and more use of ORM models, it is more convenient to have access to an `AsyncSession` object which can deal with those models. + Eventually, when verification against PHP REST API output is no longer necessary, we do not even need the `AsyncConnection` objects anymore at all. + + +Above fixtures have considerable per-test overhead. Use them only when you need them. More details in the next section. + ### Writing Tests for an Endpoint Because the `py_api` and database fixtures provide considerable per-test overhead, follow these guidelines for writing a test suite for an endpoint. +!!! warning "Code snippets may not work" + + The code below is provided as a guide, but isn't automatically tested (yet). This means it may be out of sync. + For examples that work, reference our test suite. + Include tests against `py_api` for input validation specific to that endpoint. Validation in reused components should be tested centrally (e.g., Pagination). ```python def test_get_dataset_identifier_validation(py_api: httpx.AsyncClient) -> None: @@ -102,17 +142,17 @@ def test_get_dataset_success(py_api: httpx.AsyncClient) -> None: For all other tests, do not use `py_api` but call the implementing function directly. For example, do not call `client.get("/datasets/1")` but instead `get_dataset`: ```python -async def test_get_dataset_private_success(expdb_test: AsyncConnection, user_test: AsyncConnection) -> None: +async def test_get_dataset_private_success(expdb_session: AsyncSession, userdb_session: AsyncSession) -> None: private_dataset = 42 owner_of_that_dataset = OWNER_USER - dataset = await get_dataset(dataset_id=42, user=owner_of_that_dataset, user_db=user_test, expdb_db=expdb_test) + dataset = await get_dataset(dataset_id=42, user=owner_of_that_dataset, userdb_session=userdb_session, expdb_session=expdb_session) assert dataset.id == private_dataset -async def test_get_dataset_private_access_denied(expdb_test: AsyncConnection, user_test: AsyncConnection) -> None: +async def test_get_dataset_private_access_denied(expdb_session: AsyncSession, userdb_session: AsyncSession) -> None: private_dataset = 42 owner_of_that_dataset = SOME_USER # Test User defined in a common file with pytest.raises(DatasetNoAccessError) as e: - await get_dataset(dataset_id=42, user=owner_of_that_dataset, user_db=user_test, expdb_db=expdb_test) + await get_dataset(dataset_id=42, user=owner_of_that_dataset, userdb_session=userdb_session, expdb_session=expdb_session) assert e.value.status_code == HTTPStatus.FORBIDDEN ``` @@ -164,7 +204,7 @@ def _assert_error_response_equal(py_response, php_response) -> None: You frequently need to write tests which include fetching from or writing to the database. There is a test database that is prepopulated with data available for use as defined in our `compose.yaml` file. -The `expdb_test` and `user_test` connections automatically start a transaction during setup and perform a rollback during teardown. +The `expdb_connection`, `userdb_connection`, `expdb_session`, and `userdb_session` fixtures automatically start a transaction during setup and perform a rollback during teardown. This means that as long as you do not `.commit()` any changes, the data will not persist. This is a good thing. We do not want our tests to have side effects, as it might lead to inconsistent behavior. diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index f8bbd1f8..d32965c1 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -9,7 +9,7 @@ from fastapi import Depends from loguru import logger from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession from core.errors import AuthenticationFailedError, AuthenticationRequiredError from database.engine import expdb_database, user_database diff --git a/src/routers/runs.py b/src/routers/runs.py index b030bea4..f23fd599 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -1,8 +1,7 @@ """Endpoints relating to Runs and Traces.""" -import asyncio from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, Any, cast +from typing import TYPE_CHECKING, Annotated, cast from fastapi import APIRouter, Depends @@ -91,12 +90,17 @@ async def _load_run_context( input_data_rows = await database.runs.get_input_data(run.rid, expdb) output_file_rows = await database.runs.get_output_files(run.rid, expdb) evaluation_rows = await database.runs.get_evaluations( - run.rid, expdb, evaluation_engine_ids=engine_ids + run.rid, + expdb, + evaluation_engine_ids=engine_ids, ) task_type = await database.tasks.get_task_type_name(run.task_id, expdb) task_evaluation_measure = await database.tasks.get_task_evaluation_measure(run.task_id, expdb) setup = await database.setups.get(run.setup, expdb) - parameter_rows = await database.setups.get_parameters(run.setup, expdb) + parameter_rows = cast( + "list[UntypedRow]", + await database.setups.get_parameters(run.setup, expdb), + ) return RunContext( uploader_name=uploader_user.full_name if uploader_user else None, tags=tags, diff --git a/tests/conftest.py b/tests/conftest.py index 354eaea9..ce9a44ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,8 +9,7 @@ import httpx import pytest from _pytest.config import Config # noqa: TC002 used during collection by Pytest -from _pytest.nodes import Item -from sqlalchemy.orm import Session # noqa: TC002 used during collection by Pytest +from _pytest.nodes import Item # noqa: TC002 used during collection by Pytest from asgi_lifespan import LifespanManager from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession @@ -27,8 +26,14 @@ from main import create_api from routers.dependencies import ( expdb_connection as expdb_connection_dep, - userdb_connection as userdb_connection_dep, +) +from routers.dependencies import ( expdb_session as expdb_session_dep, +) +from routers.dependencies import ( + userdb_connection as userdb_connection_dep, +) +from routers.dependencies import ( userdb_session as userdb_session_dep, ) from tests.users import OWNER_USER @@ -42,10 +47,11 @@ @contextlib.asynccontextmanager async def automatic_rollback(engine: AsyncEngine) -> AsyncIterator[AsyncConnection]: - async with engine.connect() as connection, connection.begin() as transaction: + async with engine.connect() as connection: + await connection.begin() yield connection - if transaction.is_active: - await transaction.rollback() + if connection.in_transaction(): + await connection.rollback() @contextlib.asynccontextmanager @@ -290,23 +296,23 @@ async def flow(expdb_session: AsyncSession) -> Flow: @pytest.fixture -async def persisted_flow(flow: Flow, expdb_session: AsyncSession) -> AsyncIterator[Flow]: - await expdb_session.commit() +async def persisted_flow(flow: Flow, expdb_connection: AsyncConnection) -> AsyncIterator[Flow]: + await expdb_connection.commit() yield flow # We want to ensure the commit below does not accidentally persist new # data to the database. - await expdb_session.rollback() + await expdb_connection.rollback() - await expdb_session.execute( + await expdb_connection.execute( text( """ DELETE FROM implementation WHERE id = :flow_id """, ), - params={"flow_id": flow.id}, + parameters={"flow_id": flow.id}, ) - await expdb_session.commit() + await expdb_connection.commit() @pytest.fixture From 19b9557b1565e09646b7ee481dc30e3c636f2305 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:30:46 +0000 Subject: [PATCH 12/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/routers/datasets.py | 2 -- src/routers/flows.py | 2 -- src/routers/tasks.py | 1 - tests/database/flows_test.py | 4 +++- tests/routers/dataset_untag_test.py | 2 +- tests/routers/datasets_get_test.py | 2 +- tests/routers/qualities_list_test.py | 2 +- tests/routers/setups_tag_test.py | 2 +- tests/routers/setups_untag_test.py | 2 +- tests/routers/study_attach_test.py | 2 +- tests/routers/users_delete_test.py | 4 +++- 11 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 68b48831..b23babdd 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -6,7 +6,6 @@ and 'qualities' (or meta-features) that describe the data (for example, number of rows or columns). """ -import asyncio import html import re from datetime import datetime @@ -19,7 +18,6 @@ from sqlalchemy import bindparam, text import database.datasets -from database.models import tags import database.qualities from config import get_config from core.access import user_has_access diff --git a/src/routers/flows.py b/src/routers/flows.py index 47a2418e..7dc1521c 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -1,6 +1,5 @@ """Defines endpoints relating to Flows.""" -import asyncio from typing import TYPE_CHECKING, Annotated, Literal from fastapi import APIRouter, Depends @@ -9,7 +8,6 @@ from core.conversions import str_to_num from core.errors import FlowNotFoundError from core.types import Identifier -from database.models import tags from routers.dependencies import expdb_session from routers.schemas.flows import Flow, Parameter, Subflow diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 27043998..7aab884a 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -1,6 +1,5 @@ """Defines endpoints relating to Tasks.""" -import asyncio import json import re from enum import StrEnum diff --git a/tests/database/flows_test.py b/tests/database/flows_test.py index 82ec06a2..617b4f02 100644 --- a/tests/database/flows_test.py +++ b/tests/database/flows_test.py @@ -8,7 +8,9 @@ async def test_database_flow_exists(flow: Flow, expdb_session: AsyncSession) -> None: - retrieved_flow = await database.flows.get_by_name(flow.name, flow.external_version, expdb_session) + retrieved_flow = await database.flows.get_by_name( + flow.name, flow.external_version, expdb_session + ) assert retrieved_flow is not None assert retrieved_flow.id == flow.id # when using actual ORM, can instead ensure _all_ fields match. diff --git a/tests/routers/dataset_untag_test.py b/tests/routers/dataset_untag_test.py index c0673cbf..b332b763 100644 --- a/tests/routers/dataset_untag_test.py +++ b/tests/routers/dataset_untag_test.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def test_dataset_untag_success( diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index 4a3317f2..6a1022ec 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def test_get_dataset_via_api(py_api: httpx.AsyncClient) -> None: diff --git a/tests/routers/qualities_list_test.py b/tests/routers/qualities_list_test.py index ba02986e..4f4b5456 100644 --- a/tests/routers/qualities_list_test.py +++ b/tests/routers/qualities_list_test.py @@ -7,7 +7,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def _remove_quality_from_database(quality_name: str, expdb_session: AsyncSession) -> None: diff --git a/tests/routers/setups_tag_test.py b/tests/routers/setups_tag_test.py index ec267008..e31077f0 100644 --- a/tests/routers/setups_tag_test.py +++ b/tests/routers/setups_tag_test.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def test_setup_tag_missing_auth(py_api: httpx.AsyncClient) -> None: diff --git a/tests/routers/setups_untag_test.py b/tests/routers/setups_untag_test.py index 39dac022..30749eb9 100644 --- a/tests/routers/setups_untag_test.py +++ b/tests/routers/setups_untag_test.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def test_setup_untag_missing_auth(py_api: httpx.AsyncClient) -> None: diff --git a/tests/routers/study_attach_test.py b/tests/routers/study_attach_test.py index c1d56593..2bb852f5 100644 --- a/tests/routers/study_attach_test.py +++ b/tests/routers/study_attach_test.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: import httpx - from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession + from sqlalchemy.ext.asyncio import AsyncSession async def _attach_tasks_to_study( diff --git a/tests/routers/users_delete_test.py b/tests/routers/users_delete_test.py index 9f049014..8069b0dc 100644 --- a/tests/routers/users_delete_test.py +++ b/tests/routers/users_delete_test.py @@ -9,7 +9,9 @@ import pytest_mock # noqa: TC002 used at runtime by pytest fixtures from sqlalchemy import text from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession # noqa: TC002 used at runtime by pytest fixtures +from sqlalchemy.ext.asyncio import ( # noqa: TC002 used at runtime by pytest fixtures + AsyncSession, +) from core.errors import AccountHasResourcesError, ForbiddenError, UserNotFoundError from core.types import Identifier From 992757beba71f19f42a7ab8778027f416b99daaa Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 11 Jul 2026 13:18:46 +0200 Subject: [PATCH 13/14] address bot review feedback --- docs/development/tests.md | 4 ++-- src/routers/tasks.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/development/tests.md b/docs/development/tests.md index 542c8160..659c776d 100644 --- a/docs/development/tests.md +++ b/docs/development/tests.md @@ -93,7 +93,7 @@ async def test_python(py_api: httpx.AsyncClient, expdb_session: AsyncSession) -> # after the test is done, the fixture clean up will ensure the change is not committed to the database, no extra code needed async def test_python_and_php(py_api: httpx.AsyncClient, php_api: httpx.AsyncClient, expdb_connection: AsyncConnection) -> None: - await expdb_connection.execute(text("INSERT INTO dataset ..."), params=...) # Insert dataset with id 42 + await expdb_connection.execute(text("INSERT INTO dataset ..."), parameters=...) # Insert dataset with id 42 await expdb_connection.commit() # We need to persist the data in the database, because the PHP REST API cannot see our transaction response = await php_api.get("/datasets/42") # The PHP REST API can see the dataset, because it exists in the database @@ -101,7 +101,7 @@ async def test_python_and_php(py_api: httpx.AsyncClient, php_api: httpx.AsyncCli # We need to clean up after ourselves, otherwise the test has side effects. # This isn't a great pattern, prefer instead the use of context managers which will execute the delete statements even if unexpected exceptions occur. - await expdb_connection.execute(text("DELETE FROM dataset ..."), params=...) + await expdb_connection.execute(text("DELETE FROM dataset ..."), parameters=...) await expdb_connection.commit() ``` diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 27043998..7e08e557 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -1,6 +1,5 @@ """Defines endpoints relating to Tasks.""" -import asyncio import json import re from enum import StrEnum @@ -471,7 +470,7 @@ async def get_task( (name, template) for name, io, required, template in templates if io == "input" ] filled_templates = [] - for name, template in input_templates: + for _, template in input_templates: filled_templates.append(await fill_template(template, task, task_inputs, expdb)) inputs = [ filled | {"name": name} From 8954d287688ef64997de85948918737afce9e2ae Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 11 Jul 2026 13:31:17 +0200 Subject: [PATCH 14/14] Remove unused connection dependencies --- src/routers/dependencies.py | 14 +------------- tests/conftest.py | 18 ------------------ 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index d32965c1..2ed817f5 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -9,25 +9,13 @@ from fastapi import Depends from loguru import logger from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession +from sqlalchemy.ext.asyncio import AsyncSession from core.errors import AuthenticationFailedError, AuthenticationRequiredError from database.engine import expdb_database, user_database from database.users import APIKey, User -async def expdb_connection() -> AsyncIterator[AsyncConnection]: - engine = expdb_database() - async with engine.connect() as connection, connection.begin(): - yield connection - - -async def userdb_connection() -> AsyncIterator[AsyncConnection]: - engine = user_database() - async with engine.connect() as connection, connection.begin(): - yield connection - - async def expdb_session() -> AsyncIterator[AsyncSession]: engine = expdb_database() async with AsyncSession(engine) as session, session.begin(): diff --git a/tests/conftest.py b/tests/conftest.py index ce9a44ae..338341f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,15 +24,9 @@ from core.types import Identifier from database.engine import expdb_database, user_database from main import create_api -from routers.dependencies import ( - expdb_connection as expdb_connection_dep, -) from routers.dependencies import ( expdb_session as expdb_session_dep, ) -from routers.dependencies import ( - userdb_connection as userdb_connection_dep, -) from routers.dependencies import ( userdb_session as userdb_session_dep, ) @@ -131,8 +125,6 @@ async def app() -> AsyncIterator[FastAPI]: @pytest.fixture async def py_api( - expdb_connection: AsyncConnection, - userdb_connection: AsyncConnection, expdb_session: AsyncSession, userdb_session: AsyncSession, app: FastAPI, @@ -146,20 +138,12 @@ async def py_api( # expdb_session.execute(...) # write some data # noqa: ERA001 # py_api.get(...) # read that data # noqa: ERA001 - async def override_expdb_connection() -> AsyncIterator[AsyncConnection]: - yield expdb_connection - - async def override_userdb_connection() -> AsyncIterator[AsyncConnection]: - yield userdb_connection - async def override_expdb_session() -> AsyncIterator[AsyncSession]: yield expdb_session async def override_userdb_session() -> AsyncIterator[AsyncSession]: yield userdb_session - app.dependency_overrides[expdb_connection_dep] = override_expdb_connection - app.dependency_overrides[userdb_connection_dep] = override_userdb_connection app.dependency_overrides[expdb_session_dep] = override_expdb_session app.dependency_overrides[userdb_session_dep] = override_userdb_session @@ -170,8 +154,6 @@ async def override_userdb_session() -> AsyncIterator[AsyncSession]: ) as client: yield client - app.dependency_overrides[expdb_connection_dep] = expdb_connection_dep - app.dependency_overrides[userdb_connection_dep] = userdb_connection_dep app.dependency_overrides[expdb_session_dep] = expdb_session_dep app.dependency_overrides[userdb_session_dep] = userdb_session_dep