From 37e2476097cbf24281058e5549e88e80943b784c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 15:21:02 +0200 Subject: [PATCH 01/10] Rename the `schema` directory to the more conventional `models` --- src/core/formatting.py | 2 +- src/database/datasets.py | 2 +- src/database/evaluations.py | 2 +- src/database/flows.py | 2 +- src/database/{schema => models}/__init__.py | 0 src/database/{schema => models}/base.py | 0 src/database/{schema => models}/setups.py | 2 +- src/database/{schema => models}/tags.py | 2 +- src/database/runs.py | 2 +- src/database/setups.py | 4 ++-- src/database/studies.py | 2 +- src/database/tasks.py | 4 ++-- src/main.py | 2 +- src/routers/openml/runs.py | 4 ++-- src/routers/openml/study.py | 2 +- src/routers/openml/tasks.py | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) rename src/database/{schema => models}/__init__.py (100%) rename src/database/{schema => models}/base.py (100%) rename src/database/{schema => models}/setups.py (88%) rename src/database/{schema => models}/tags.py (95%) diff --git a/src/core/formatting.py b/src/core/formatting.py index a4b09e8..832d9aa 100644 --- a/src/core/formatting.py +++ b/src/core/formatting.py @@ -1,7 +1,7 @@ import html from config import get_config -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from schemas.datasets.openml import DatasetFileFormat diff --git a/src/database/datasets.py b/src/database/datasets.py index 5cb99d0..3cdd23d 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -13,7 +13,7 @@ DuplicatePrimaryKeyError, ForeignKeyConstraintError, ) -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from routers.types import Identifier, TagString from schemas.datasets.openml import DatasetStatus, Feature diff --git a/src/database/evaluations.py b/src/database/evaluations.py index 14bec4d..4ac1f40 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -4,7 +4,7 @@ from sqlalchemy import text from core.formatting import _str_to_bool -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from schemas.datasets.openml import EstimationProcedure if TYPE_CHECKING: diff --git a/src/database/flows.py b/src/database/flows.py index 830a386..84962a3 100644 --- a/src/database/flows.py +++ b/src/database/flows.py @@ -3,7 +3,7 @@ from sqlalchemy import text -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from routers.types import Identifier if TYPE_CHECKING: diff --git a/src/database/schema/__init__.py b/src/database/models/__init__.py similarity index 100% rename from src/database/schema/__init__.py rename to src/database/models/__init__.py diff --git a/src/database/schema/base.py b/src/database/models/base.py similarity index 100% rename from src/database/schema/base.py rename to src/database/models/base.py diff --git a/src/database/schema/setups.py b/src/database/models/setups.py similarity index 88% rename from src/database/schema/setups.py rename to src/database/models/setups.py index 36f9669..568c6ff 100644 --- a/src/database/schema/setups.py +++ b/src/database/models/setups.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Mapped, mapped_column -from database.schema.base import Base, ExpDBReflected +from database.models.base import Base, ExpDBReflected from routers.types import Identifier diff --git a/src/database/schema/tags.py b/src/database/models/tags.py similarity index 95% rename from src/database/schema/tags.py rename to src/database/models/tags.py index 5fd687e..4c42858 100644 --- a/src/database/schema/tags.py +++ b/src/database/models/tags.py @@ -5,7 +5,7 @@ from sqlalchemy import FetchedValue from sqlalchemy.orm import Mapped, mapped_column -from database.schema.base import Base, ExpDBReflected +from database.models.base import Base, ExpDBReflected from routers.types import Identifier, TagString diff --git a/src/database/runs.py b/src/database/runs.py index 0c5c93c..d554fd6 100644 --- a/src/database/runs.py +++ b/src/database/runs.py @@ -5,7 +5,7 @@ from sqlalchemy import bindparam, text -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from routers.types import Identifier if TYPE_CHECKING: diff --git a/src/database/setups.py b/src/database/setups.py index 2bd20f8..5dceadd 100644 --- a/src/database/setups.py +++ b/src/database/setups.py @@ -12,8 +12,8 @@ DuplicatePrimaryKeyError, ForeignKeyConstraintError, ) -from database.schema.setups import Setup -from database.schema.tags import SetupTag +from database.models.setups import Setup +from database.models.tags import SetupTag from routers.types import Identifier, TagString if TYPE_CHECKING: diff --git a/src/database/studies.py b/src/database/studies.py index b4a6936..6f9d16d 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -5,7 +5,7 @@ from sqlalchemy import text -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from database.users import User from routers.types import Identifier from schemas.study import CreateStudy, StudyType diff --git a/src/database/tasks.py b/src/database/tasks.py index ae8aaa0..c7860ed 100644 --- a/src/database/tasks.py +++ b/src/database/tasks.py @@ -10,8 +10,8 @@ DuplicatePrimaryKeyError, ForeignKeyConstraintError, ) -from database.schema.base import UntypedRow -from database.schema.tags import TaskTag +from database.models.base import UntypedRow +from database.models.tags import TaskTag from routers.types import Identifier, TagString if TYPE_CHECKING: diff --git a/src/main.py b/src/main.py index 6207b15..585b2f9 100644 --- a/src/main.py +++ b/src/main.py @@ -26,7 +26,7 @@ request_response_logger, setup_log_sinks, ) -from database.schema.base import reflect_db_schemas +from database.models.base import reflect_db_schemas from database.setup import close_databases from routers.openml.datasets import router as datasets_router from routers.openml.estimation_procedure import router as estimationprocedure_router diff --git a/src/routers/openml/runs.py b/src/routers/openml/runs.py index 3869df8..26ae92d 100644 --- a/src/routers/openml/runs.py +++ b/src/routers/openml/runs.py @@ -13,8 +13,8 @@ import database.tasks import database.users from core.errors import RunNotFoundError, RunTraceNotFoundError -from database.schema.base import UntypedRow -from database.schema.setups import Setup +from database.models.base import UntypedRow +from database.models.setups import Setup from routers.dependencies import expdb_connection, expdb_session, userdb_connection from routers.types import Identifier from schemas.runs import ( diff --git a/src/routers/openml/study.py b/src/routers/openml/study.py index dbb6581..7f2fd20 100644 --- a/src/routers/openml/study.py +++ b/src/routers/openml/study.py @@ -16,7 +16,7 @@ StudyPrivateError, ) from core.formatting import _str_to_bool -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from database.users import User from routers.dependencies import expdb_connection, fetch_user, fetch_user_or_raise from routers.types import Identifier diff --git a/src/routers/openml/tasks.py b/src/routers/openml/tasks.py index 55df619..b46c017 100644 --- a/src/routers/openml/tasks.py +++ b/src/routers/openml/tasks.py @@ -14,7 +14,7 @@ from config import get_config from core.errors import InternalError, NoResultsError, TagAlreadyExistsError, TaskNotFoundError from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError -from database.schema.base import UntypedRow +from database.models.base import UntypedRow from database.users import User from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise from routers.types import ( From a6179184971e605a18bbc5c9c23a102d50f017d1 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 15:27:11 +0200 Subject: [PATCH 02/10] Move the types to `core` --- src/{routers => core}/types.py | 2 ++ src/database/datasets.py | 2 +- src/database/flows.py | 2 +- src/database/models/setups.py | 2 +- src/database/models/tags.py | 2 +- src/database/qualities.py | 2 +- src/database/runs.py | 2 +- src/database/setups.py | 2 +- src/database/studies.py | 2 +- src/database/tasks.py | 2 +- src/database/users.py | 2 +- src/routers/openml/datasets.py | 14 +++++++------- src/routers/openml/flows.py | 2 +- src/routers/openml/qualities.py | 2 +- src/routers/openml/runs.py | 2 +- src/routers/openml/setups.py | 2 +- src/routers/openml/study.py | 2 +- src/routers/openml/tasks.py | 10 +++++----- tests/conftest.py | 2 +- tests/types_test.py | 2 +- 20 files changed, 31 insertions(+), 29 deletions(-) rename src/{routers => core}/types.py (92%) diff --git a/src/routers/types.py b/src/core/types.py similarity index 92% rename from src/routers/types.py rename to src/core/types.py index fe94577..6d33ec9 100644 --- a/src/routers/types.py +++ b/src/core/types.py @@ -1,3 +1,5 @@ +"""Common types used in the definition of schemas and models.""" + from typing import Annotated from pydantic import Field diff --git a/src/database/datasets.py b/src/database/datasets.py index 3cdd23d..312986e 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -7,6 +7,7 @@ from sqlalchemy import text from sqlalchemy.exc import IntegrityError +from core.types import Identifier, TagString from database.exceptions import ( _DUPLICATE_ENTRY, _FOREIGN_KEY_CONSTRAINT_FAILED, @@ -14,7 +15,6 @@ ForeignKeyConstraintError, ) from database.models.base import UntypedRow -from routers.types import Identifier, TagString from schemas.datasets.openml import DatasetStatus, Feature if TYPE_CHECKING: diff --git a/src/database/flows.py b/src/database/flows.py index 84962a3..d245d66 100644 --- a/src/database/flows.py +++ b/src/database/flows.py @@ -3,8 +3,8 @@ from sqlalchemy import text +from core.types import Identifier from database.models.base import UntypedRow -from routers.types import Identifier if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/models/setups.py b/src/database/models/setups.py index 568c6ff..3ff94d2 100644 --- a/src/database/models/setups.py +++ b/src/database/models/setups.py @@ -2,8 +2,8 @@ from sqlalchemy.orm import Mapped, mapped_column +from core.types import Identifier from database.models.base import Base, ExpDBReflected -from routers.types import Identifier class Setup(ExpDBReflected, Base): diff --git a/src/database/models/tags.py b/src/database/models/tags.py index 4c42858..560b680 100644 --- a/src/database/models/tags.py +++ b/src/database/models/tags.py @@ -5,8 +5,8 @@ from sqlalchemy import FetchedValue from sqlalchemy.orm import Mapped, mapped_column +from core.types import Identifier, TagString from database.models.base import Base, ExpDBReflected -from routers.types import Identifier, TagString class Tag: diff --git a/src/database/qualities.py b/src/database/qualities.py index 4899f02..81bc1d4 100644 --- a/src/database/qualities.py +++ b/src/database/qualities.py @@ -4,7 +4,7 @@ from sqlalchemy import text -from routers.types import Identifier +from core.types import Identifier from schemas.datasets.openml import Quality if TYPE_CHECKING: diff --git a/src/database/runs.py b/src/database/runs.py index d554fd6..2a4d547 100644 --- a/src/database/runs.py +++ b/src/database/runs.py @@ -5,8 +5,8 @@ from sqlalchemy import bindparam, text +from core.types import Identifier from database.models.base import UntypedRow -from routers.types import Identifier if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/setups.py b/src/database/setups.py index 5dceadd..2808cd9 100644 --- a/src/database/setups.py +++ b/src/database/setups.py @@ -6,6 +6,7 @@ from sqlalchemy import select, text from sqlalchemy.exc import IntegrityError +from core.types import Identifier, TagString from database.exceptions import ( _DUPLICATE_ENTRY, _FOREIGN_KEY_CONSTRAINT_FAILED, @@ -14,7 +15,6 @@ ) from database.models.setups import Setup from database.models.tags import SetupTag -from routers.types import Identifier, TagString if TYPE_CHECKING: from sqlalchemy.engine import RowMapping diff --git a/src/database/studies.py b/src/database/studies.py index 6f9d16d..b961716 100644 --- a/src/database/studies.py +++ b/src/database/studies.py @@ -5,9 +5,9 @@ from sqlalchemy import text +from core.types import Identifier from database.models.base import UntypedRow from database.users import User -from routers.types import Identifier from schemas.study import CreateStudy, StudyType if TYPE_CHECKING: diff --git a/src/database/tasks.py b/src/database/tasks.py index c7860ed..74c330b 100644 --- a/src/database/tasks.py +++ b/src/database/tasks.py @@ -4,6 +4,7 @@ from sqlalchemy import select, text from sqlalchemy.exc import IntegrityError +from core.types import Identifier, TagString from database.exceptions import ( _DUPLICATE_ENTRY, _FOREIGN_KEY_CONSTRAINT_FAILED, @@ -12,7 +13,6 @@ ) from database.models.base import UntypedRow from database.models.tags import TaskTag -from routers.types import Identifier, TagString if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession diff --git a/src/database/users.py b/src/database/users.py index 745e311..3e46ab0 100644 --- a/src/database/users.py +++ b/src/database/users.py @@ -8,7 +8,7 @@ from sqlalchemy import text from config import get_config -from routers.types import Identifier +from core.types import Identifier if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/openml/datasets.py b/src/routers/openml/datasets.py index 72c94de..b741f4d 100644 --- a/src/routers/openml/datasets.py +++ b/src/routers/openml/datasets.py @@ -33,6 +33,13 @@ _format_dataset_url, _format_parquet_url, ) +from core.types import ( + CasualString128, + Identifier, + IntegerRange, + TagString, + integer_range_regex, +) from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError from database.users import User from routers.dependencies import ( @@ -42,13 +49,6 @@ fetch_user_or_raise, userdb_connection, ) -from routers.types import ( - CasualString128, - Identifier, - IntegerRange, - TagString, - integer_range_regex, -) from schemas.datasets.openml import DatasetMetadata, DatasetStatus, Feature, FeatureType if TYPE_CHECKING: diff --git a/src/routers/openml/flows.py b/src/routers/openml/flows.py index b3c3f21..1df5f08 100644 --- a/src/routers/openml/flows.py +++ b/src/routers/openml/flows.py @@ -6,8 +6,8 @@ import database.flows from core.conversions import _str_to_num from core.errors import FlowNotFoundError +from core.types import Identifier from routers.dependencies import expdb_connection -from routers.types import Identifier from schemas.flows import Flow, Parameter, Subflow if TYPE_CHECKING: diff --git a/src/routers/openml/qualities.py b/src/routers/openml/qualities.py index dc7a3f5..f863fc2 100644 --- a/src/routers/openml/qualities.py +++ b/src/routers/openml/qualities.py @@ -11,9 +11,9 @@ DatasetProcessingError, NoQualitiesError, ) +from core.types import Identifier from database.users import User from routers.dependencies import expdb_connection, fetch_user -from routers.types import Identifier from schemas.datasets.openml import Quality if TYPE_CHECKING: diff --git a/src/routers/openml/runs.py b/src/routers/openml/runs.py index 26ae92d..53ba5fa 100644 --- a/src/routers/openml/runs.py +++ b/src/routers/openml/runs.py @@ -13,10 +13,10 @@ import database.tasks import database.users from core.errors import RunNotFoundError, RunTraceNotFoundError +from core.types import Identifier from database.models.base import UntypedRow from database.models.setups import Setup from routers.dependencies import expdb_connection, expdb_session, userdb_connection -from routers.types import Identifier from schemas.runs import ( EvaluationScore, InputDataset, diff --git a/src/routers/openml/setups.py b/src/routers/openml/setups.py index d7d8694..615f3fb 100644 --- a/src/routers/openml/setups.py +++ b/src/routers/openml/setups.py @@ -12,10 +12,10 @@ TagNotFoundError, TagNotOwnedError, ) +from core.types import Identifier, TagString from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError from database.users import User from routers.dependencies import expdb_connection, expdb_session, fetch_user_or_raise -from routers.types import Identifier, TagString from schemas.setups import SetupParameters, SetupResponse if TYPE_CHECKING: diff --git a/src/routers/openml/study.py b/src/routers/openml/study.py index 7f2fd20..6cbdde8 100644 --- a/src/routers/openml/study.py +++ b/src/routers/openml/study.py @@ -16,10 +16,10 @@ StudyPrivateError, ) from core.formatting import _str_to_bool +from core.types import Identifier from database.models.base import UntypedRow from database.users import User from routers.dependencies import expdb_connection, fetch_user, fetch_user_or_raise -from routers.types import Identifier from schemas.core import Visibility from schemas.study import CreateStudy, Study, StudyStatus, StudyType diff --git a/src/routers/openml/tasks.py b/src/routers/openml/tasks.py index b46c017..16353f0 100644 --- a/src/routers/openml/tasks.py +++ b/src/routers/openml/tasks.py @@ -13,17 +13,17 @@ import database.tasks from config import get_config from core.errors import InternalError, NoResultsError, TagAlreadyExistsError, TaskNotFoundError -from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError -from database.models.base import UntypedRow -from database.users import User -from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise -from routers.types import ( +from core.types import ( CasualString128, Identifier, IntegerRange, TagString, integer_range_regex, ) +from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError +from database.models.base import UntypedRow +from database.users import User +from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise from schemas.datasets.openml import Task if TYPE_CHECKING: diff --git a/tests/conftest.py b/tests/conftest.py index 68a8e1c..035af5c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,10 +21,10 @@ LoggingConfiguration, RoutingConfiguration, ) +from core.types import Identifier from database.setup import expdb_database, user_database from main import create_api from routers.dependencies import expdb_connection, userdb_connection -from routers.types import Identifier from tests.users import OWNER_USER if TYPE_CHECKING: diff --git a/tests/types_test.py b/tests/types_test.py index 7a4df61..2ded54c 100644 --- a/tests/types_test.py +++ b/tests/types_test.py @@ -11,7 +11,7 @@ import pytest from pydantic import TypeAdapter, ValidationError -from routers.types import CasualString, Identifier, TagString +from core.types import CasualString, Identifier, TagString _identifier = TypeAdapter(Identifier) From ebe229a190c0b43595ee47d8e591e137fb54b949 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 15:41:16 +0200 Subject: [PATCH 03/10] Remove the `openml` submodule structure --- src/main.py | 22 +++++++++---------- src/routers/__init__.py | 2 +- src/routers/{openml => }/datasets.py | 0 .../{openml => }/estimation_procedure.py | 0 src/routers/{openml => }/evaluations.py | 0 src/routers/{openml => }/flows.py | 0 src/routers/openml/__init__.py | 1 - src/routers/{openml => }/qualities.py | 0 src/routers/{openml => }/runs.py | 0 src/routers/{openml => }/setups.py | 0 src/routers/{openml => }/study.py | 0 src/routers/{openml => }/tasks.py | 0 src/routers/{openml => }/tasktype.py | 0 src/routers/{openml => }/users.py | 0 .../routers/{openml => }/dataset_tag_test.py | 4 ++-- .../{openml => }/dataset_untag_test.py | 2 +- .../{openml => }/datasets_features_test.py | 2 +- .../routers/{openml => }/datasets_get_test.py | 2 +- .../datasets_list_datasets_test.py | 2 +- .../{openml => }/datasets_qualities_test.py | 0 .../{openml => }/datasets_status_test.py | 2 +- .../{openml => }/estimation_procedure_test.py | 0 .../{openml => }/evaluation_measure_test.py | 0 .../routers/{openml => }/flows_exists_test.py | 2 +- tests/routers/{openml => }/flows_get_test.py | 0 tests/routers/openml/__init__.py | 0 .../{openml => }/qualities_list_test.py | 0 tests/routers/{openml => }/runs_get_test.py | 6 ++--- tests/routers/{openml => }/runs_trace_test.py | 0 tests/routers/{openml => }/setups_get_test.py | 0 tests/routers/{openml => }/setups_tag_test.py | 2 +- .../routers/{openml => }/setups_untag_test.py | 2 +- .../routers/{openml => }/study_attach_test.py | 0 tests/routers/{openml => }/study_get_test.py | 0 tests/routers/{openml => }/study_post_test.py | 0 tests/routers/{openml => }/tag_test_helper.py | 0 tests/routers/{openml => }/task_get_test.py | 0 tests/routers/{openml => }/task_list_test.py | 2 +- tests/routers/{openml => }/task_tag_test.py | 4 ++-- .../{openml => }/task_type_get_test.py | 0 .../{openml => }/task_type_list_test.py | 0 .../routers/{openml => }/users_delete_test.py | 6 ++--- 42 files changed, 31 insertions(+), 32 deletions(-) rename src/routers/{openml => }/datasets.py (100%) rename src/routers/{openml => }/estimation_procedure.py (100%) rename src/routers/{openml => }/evaluations.py (100%) rename src/routers/{openml => }/flows.py (100%) delete mode 100644 src/routers/openml/__init__.py rename src/routers/{openml => }/qualities.py (100%) rename src/routers/{openml => }/runs.py (100%) rename src/routers/{openml => }/setups.py (100%) rename src/routers/{openml => }/study.py (100%) rename src/routers/{openml => }/tasks.py (100%) rename src/routers/{openml => }/tasktype.py (100%) rename src/routers/{openml => }/users.py (100%) rename tests/routers/{openml => }/dataset_tag_test.py (96%) rename tests/routers/{openml => }/dataset_untag_test.py (99%) rename tests/routers/{openml => }/datasets_features_test.py (99%) rename tests/routers/{openml => }/datasets_get_test.py (99%) rename tests/routers/{openml => }/datasets_list_datasets_test.py (99%) rename tests/routers/{openml => }/datasets_qualities_test.py (100%) rename tests/routers/{openml => }/datasets_status_test.py (98%) rename tests/routers/{openml => }/estimation_procedure_test.py (100%) rename tests/routers/{openml => }/evaluation_measure_test.py (100%) rename tests/routers/{openml => }/flows_exists_test.py (98%) rename tests/routers/{openml => }/flows_get_test.py (100%) delete mode 100644 tests/routers/openml/__init__.py rename tests/routers/{openml => }/qualities_list_test.py (100%) rename tests/routers/{openml => }/runs_get_test.py (97%) rename tests/routers/{openml => }/runs_trace_test.py (100%) rename tests/routers/{openml => }/setups_get_test.py (100%) rename tests/routers/{openml => }/setups_tag_test.py (99%) rename tests/routers/{openml => }/setups_untag_test.py (99%) rename tests/routers/{openml => }/study_attach_test.py (100%) rename tests/routers/{openml => }/study_get_test.py (100%) rename tests/routers/{openml => }/study_post_test.py (100%) rename tests/routers/{openml => }/tag_test_helper.py (100%) rename tests/routers/{openml => }/task_get_test.py (100%) rename tests/routers/{openml => }/task_list_test.py (99%) rename tests/routers/{openml => }/task_tag_test.py (96%) rename tests/routers/{openml => }/task_type_get_test.py (100%) rename tests/routers/{openml => }/task_type_list_test.py (100%) rename tests/routers/{openml => }/users_delete_test.py (97%) diff --git a/src/main.py b/src/main.py index 585b2f9..1d80e53 100644 --- a/src/main.py +++ b/src/main.py @@ -28,17 +28,17 @@ ) from database.models.base import reflect_db_schemas from database.setup import close_databases -from routers.openml.datasets import router as datasets_router -from routers.openml.estimation_procedure import router as estimationprocedure_router -from routers.openml.evaluations import router as evaluationmeasures_router -from routers.openml.flows import router as flows_router -from routers.openml.qualities import router as qualities_router -from routers.openml.runs import router as run_router -from routers.openml.setups import router as setup_router -from routers.openml.study import router as study_router -from routers.openml.tasks import router as task_router -from routers.openml.tasktype import router as ttype_router -from routers.openml.users import router as users_router +from routers.datasets import router as datasets_router +from routers.estimation_procedure import router as estimationprocedure_router +from routers.evaluations import router as evaluationmeasures_router +from routers.flows import router as flows_router +from routers.qualities import router as qualities_router +from routers.runs import router as run_router +from routers.setups import router as setup_router +from routers.study import router as study_router +from routers.tasks import router as task_router +from routers.tasktype import router as ttype_router +from routers.users import router as users_router @asynccontextmanager diff --git a/src/routers/__init__.py b/src/routers/__init__.py index 21016d1..5d924fc 100644 --- a/src/routers/__init__.py +++ b/src/routers/__init__.py @@ -1 +1 @@ -"""API routers for the OpenML server API.""" +"""Routers for the OpenML API.""" diff --git a/src/routers/openml/datasets.py b/src/routers/datasets.py similarity index 100% rename from src/routers/openml/datasets.py rename to src/routers/datasets.py diff --git a/src/routers/openml/estimation_procedure.py b/src/routers/estimation_procedure.py similarity index 100% rename from src/routers/openml/estimation_procedure.py rename to src/routers/estimation_procedure.py diff --git a/src/routers/openml/evaluations.py b/src/routers/evaluations.py similarity index 100% rename from src/routers/openml/evaluations.py rename to src/routers/evaluations.py diff --git a/src/routers/openml/flows.py b/src/routers/flows.py similarity index 100% rename from src/routers/openml/flows.py rename to src/routers/flows.py diff --git a/src/routers/openml/__init__.py b/src/routers/openml/__init__.py deleted file mode 100644 index 5d924fc..0000000 --- a/src/routers/openml/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Routers for the OpenML API.""" diff --git a/src/routers/openml/qualities.py b/src/routers/qualities.py similarity index 100% rename from src/routers/openml/qualities.py rename to src/routers/qualities.py diff --git a/src/routers/openml/runs.py b/src/routers/runs.py similarity index 100% rename from src/routers/openml/runs.py rename to src/routers/runs.py diff --git a/src/routers/openml/setups.py b/src/routers/setups.py similarity index 100% rename from src/routers/openml/setups.py rename to src/routers/setups.py diff --git a/src/routers/openml/study.py b/src/routers/study.py similarity index 100% rename from src/routers/openml/study.py rename to src/routers/study.py diff --git a/src/routers/openml/tasks.py b/src/routers/tasks.py similarity index 100% rename from src/routers/openml/tasks.py rename to src/routers/tasks.py diff --git a/src/routers/openml/tasktype.py b/src/routers/tasktype.py similarity index 100% rename from src/routers/openml/tasktype.py rename to src/routers/tasktype.py diff --git a/src/routers/openml/users.py b/src/routers/users.py similarity index 100% rename from src/routers/openml/users.py rename to src/routers/users.py diff --git a/tests/routers/openml/dataset_tag_test.py b/tests/routers/dataset_tag_test.py similarity index 96% rename from tests/routers/openml/dataset_tag_test.py rename to tests/routers/dataset_tag_test.py index dd80966..6c8201f 100644 --- a/tests/routers/openml/dataset_tag_test.py +++ b/tests/routers/dataset_tag_test.py @@ -6,10 +6,10 @@ from core.errors import DatasetNotFoundError, TagAlreadyExistsError from database.datasets import get_tags_for from database.users import User -from routers.openml.datasets import tag_dataset +from routers.datasets import tag_dataset from tests import constants from tests.conftest import DatasetFactory -from tests.routers.openml.tag_test_helper import assert_tag_response_is_identical +from tests.routers.tag_test_helper import assert_tag_response_is_identical from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/openml/dataset_untag_test.py b/tests/routers/dataset_untag_test.py similarity index 99% rename from tests/routers/openml/dataset_untag_test.py rename to tests/routers/dataset_untag_test.py index cfaaf24..7eca889 100644 --- a/tests/routers/openml/dataset_untag_test.py +++ b/tests/routers/dataset_untag_test.py @@ -20,7 +20,7 @@ from sqlalchemy import text from core.errors import DatasetNotFoundError, TagNotFoundError, TagNotOwnedError -from routers.openml.datasets import untag_dataset +from routers.datasets import untag_dataset from tests.users import ADMIN_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/openml/datasets_features_test.py b/tests/routers/datasets_features_test.py similarity index 99% rename from tests/routers/openml/datasets_features_test.py rename to tests/routers/datasets_features_test.py index 3616659..f4e56d2 100644 --- a/tests/routers/openml/datasets_features_test.py +++ b/tests/routers/datasets_features_test.py @@ -9,7 +9,7 @@ from core.errors import DatasetNoAccessError, DatasetNotFoundError, DatasetProcessingError from database.users import User -from routers.openml.datasets import get_dataset_features +from routers.datasets import get_dataset_features from tests.users import ADMIN_USER, DATASET_130_OWNER if TYPE_CHECKING: diff --git a/tests/routers/openml/datasets_get_test.py b/tests/routers/datasets_get_test.py similarity index 99% rename from tests/routers/openml/datasets_get_test.py rename to tests/routers/datasets_get_test.py index 3ef034e..bfede5e 100644 --- a/tests/routers/openml/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -12,7 +12,7 @@ import tests.constants from core.errors import DatasetNoAccessError, DatasetNotFoundError from database.users import User -from routers.openml.datasets import get_dataset +from routers.datasets import get_dataset from schemas.datasets.openml import DatasetMetadata from tests.users import ADMIN_USER, DATASET_130_OWNER, NO_USER, SOME_USER, ApiKey diff --git a/tests/routers/openml/datasets_list_datasets_test.py b/tests/routers/datasets_list_datasets_test.py similarity index 99% rename from tests/routers/openml/datasets_list_datasets_test.py rename to tests/routers/datasets_list_datasets_test.py index 868e9ee..89ec474 100644 --- a/tests/routers/openml/datasets_list_datasets_test.py +++ b/tests/routers/datasets_list_datasets_test.py @@ -10,8 +10,8 @@ from core.errors import NoResultsError from database.users import User +from routers.datasets import DatasetStatusFilter, list_datasets from routers.dependencies import LIMIT_DEFAULT, Pagination -from routers.openml.datasets import DatasetStatusFilter, list_datasets from tests import constants from tests.users import ADMIN_USER, DATASET_130_OWNER, SOME_USER, ApiKey diff --git a/tests/routers/openml/datasets_qualities_test.py b/tests/routers/datasets_qualities_test.py similarity index 100% rename from tests/routers/openml/datasets_qualities_test.py rename to tests/routers/datasets_qualities_test.py diff --git a/tests/routers/openml/datasets_status_test.py b/tests/routers/datasets_status_test.py similarity index 98% rename from tests/routers/openml/datasets_status_test.py rename to tests/routers/datasets_status_test.py index adc5892..bee25db 100644 --- a/tests/routers/openml/datasets_status_test.py +++ b/tests/routers/datasets_status_test.py @@ -6,7 +6,7 @@ import pytest from core.errors import DatasetAdminOnlyError, DatasetNotOwnedError -from routers.openml.datasets import update_dataset_status +from routers.datasets import update_dataset_status from schemas.datasets.openml import DatasetStatus from tests import constants from tests.users import ADMIN_USER, SOME_USER diff --git a/tests/routers/openml/estimation_procedure_test.py b/tests/routers/estimation_procedure_test.py similarity index 100% rename from tests/routers/openml/estimation_procedure_test.py rename to tests/routers/estimation_procedure_test.py diff --git a/tests/routers/openml/evaluation_measure_test.py b/tests/routers/evaluation_measure_test.py similarity index 100% rename from tests/routers/openml/evaluation_measure_test.py rename to tests/routers/evaluation_measure_test.py diff --git a/tests/routers/openml/flows_exists_test.py b/tests/routers/flows_exists_test.py similarity index 98% rename from tests/routers/openml/flows_exists_test.py rename to tests/routers/flows_exists_test.py index 988402f..c588941 100644 --- a/tests/routers/openml/flows_exists_test.py +++ b/tests/routers/flows_exists_test.py @@ -6,7 +6,7 @@ import pytest from core.errors import FlowNotFoundError -from routers.openml.flows import flow_exists +from routers.flows import flow_exists from tests.conftest import Flow if TYPE_CHECKING: diff --git a/tests/routers/openml/flows_get_test.py b/tests/routers/flows_get_test.py similarity index 100% rename from tests/routers/openml/flows_get_test.py rename to tests/routers/flows_get_test.py diff --git a/tests/routers/openml/__init__.py b/tests/routers/openml/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/routers/openml/qualities_list_test.py b/tests/routers/qualities_list_test.py similarity index 100% rename from tests/routers/openml/qualities_list_test.py rename to tests/routers/qualities_list_test.py diff --git a/tests/routers/openml/runs_get_test.py b/tests/routers/runs_get_test.py similarity index 97% rename from tests/routers/openml/runs_get_test.py rename to tests/routers/runs_get_test.py index af841f6..f9d897c 100644 --- a/tests/routers/openml/runs_get_test.py +++ b/tests/routers/runs_get_test.py @@ -10,7 +10,7 @@ import pytest from core.conversions import nested_num_to_str, nested_remove_single_element_list -from routers.openml.runs import _build_evaluations +from routers.runs import _build_evaluations # ── Fixtures assume run 24 exists in the test DB (confirmed in research) ── _RUN_ID = 24 @@ -149,7 +149,7 @@ class MockRunRow(NamedTuple): error_message="Some error from the backend", ) - with patch("routers.openml.runs.database.runs.get", new_callable=AsyncMock) as mock_get: + with patch("routers.runs.database.runs.get", new_callable=AsyncMock) as mock_get: mock_get.return_value = mock_row response = await py_api.get(f"/run/{_RUN_ID}") assert response.status_code == HTTPStatus.OK @@ -181,7 +181,7 @@ async def test_task_evaluation_measure_present_when_configured( """task_evaluation_measure is present and matches DB when a measure is configured.""" # Since the test database does not have a run with an evaluation measure, we mock the DB fetch with patch( - "routers.openml.runs.database.tasks.get_task_evaluation_measure", new_callable=AsyncMock + "routers.runs.database.tasks.get_task_evaluation_measure", new_callable=AsyncMock ) as mock_get_measure: mock_get_measure.return_value = "predictive_accuracy" response = await py_api.get(f"/run/{_RUN_ID}") diff --git a/tests/routers/openml/runs_trace_test.py b/tests/routers/runs_trace_test.py similarity index 100% rename from tests/routers/openml/runs_trace_test.py rename to tests/routers/runs_trace_test.py diff --git a/tests/routers/openml/setups_get_test.py b/tests/routers/setups_get_test.py similarity index 100% rename from tests/routers/openml/setups_get_test.py rename to tests/routers/setups_get_test.py diff --git a/tests/routers/openml/setups_tag_test.py b/tests/routers/setups_tag_test.py similarity index 99% rename from tests/routers/openml/setups_tag_test.py rename to tests/routers/setups_tag_test.py index 60488c2..49bad89 100644 --- a/tests/routers/openml/setups_tag_test.py +++ b/tests/routers/setups_tag_test.py @@ -9,7 +9,7 @@ from sqlalchemy import text from core.errors import SetupNotFoundError, TagAlreadyExistsError -from routers.openml.setups import tag_setup +from routers.setups import tag_setup from tests.users import SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/openml/setups_untag_test.py b/tests/routers/setups_untag_test.py similarity index 99% rename from tests/routers/openml/setups_untag_test.py rename to tests/routers/setups_untag_test.py index c4424f1..1259f1e 100644 --- a/tests/routers/openml/setups_untag_test.py +++ b/tests/routers/setups_untag_test.py @@ -9,7 +9,7 @@ from sqlalchemy import text from core.errors import SetupNotFoundError, TagNotFoundError, TagNotOwnedError -from routers.openml.setups import untag_setup +from routers.setups import untag_setup from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/openml/study_attach_test.py b/tests/routers/study_attach_test.py similarity index 100% rename from tests/routers/openml/study_attach_test.py rename to tests/routers/study_attach_test.py diff --git a/tests/routers/openml/study_get_test.py b/tests/routers/study_get_test.py similarity index 100% rename from tests/routers/openml/study_get_test.py rename to tests/routers/study_get_test.py diff --git a/tests/routers/openml/study_post_test.py b/tests/routers/study_post_test.py similarity index 100% rename from tests/routers/openml/study_post_test.py rename to tests/routers/study_post_test.py diff --git a/tests/routers/openml/tag_test_helper.py b/tests/routers/tag_test_helper.py similarity index 100% rename from tests/routers/openml/tag_test_helper.py rename to tests/routers/tag_test_helper.py diff --git a/tests/routers/openml/task_get_test.py b/tests/routers/task_get_test.py similarity index 100% rename from tests/routers/openml/task_get_test.py rename to tests/routers/task_get_test.py diff --git a/tests/routers/openml/task_list_test.py b/tests/routers/task_list_test.py similarity index 99% rename from tests/routers/openml/task_list_test.py rename to tests/routers/task_list_test.py index 7a3f562..01443e1 100644 --- a/tests/routers/openml/task_list_test.py +++ b/tests/routers/task_list_test.py @@ -8,7 +8,7 @@ from core.conversions import nested_remove_single_element_list from core.errors import NoResultsError from routers.dependencies import LIMIT_MAX, Pagination -from routers.openml.tasks import TaskStatusFilter, list_tasks +from routers.tasks import TaskStatusFilter, list_tasks if TYPE_CHECKING: import httpx diff --git a/tests/routers/openml/task_tag_test.py b/tests/routers/task_tag_test.py similarity index 96% rename from tests/routers/openml/task_tag_test.py rename to tests/routers/task_tag_test.py index f6e04b1..8ad52b6 100644 --- a/tests/routers/openml/task_tag_test.py +++ b/tests/routers/task_tag_test.py @@ -6,10 +6,10 @@ from core.errors import TASK_NOT_FOUND_DURING_TAG, TagAlreadyExistsError, TaskNotFoundError from database.tasks import get_tags from database.users import User -from routers.openml.tasks import tag_task +from routers.tasks import tag_task from tests import constants from tests.conftest import TaskFactory -from tests.routers.openml.tag_test_helper import assert_tag_response_is_identical +from tests.routers.tag_test_helper import assert_tag_response_is_identical from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/openml/task_type_get_test.py b/tests/routers/task_type_get_test.py similarity index 100% rename from tests/routers/openml/task_type_get_test.py rename to tests/routers/task_type_get_test.py diff --git a/tests/routers/openml/task_type_list_test.py b/tests/routers/task_type_list_test.py similarity index 100% rename from tests/routers/openml/task_type_list_test.py rename to tests/routers/task_type_list_test.py diff --git a/tests/routers/openml/users_delete_test.py b/tests/routers/users_delete_test.py similarity index 97% rename from tests/routers/openml/users_delete_test.py rename to tests/routers/users_delete_test.py index 127e89e..c872c74 100644 --- a/tests/routers/openml/users_delete_test.py +++ b/tests/routers/users_delete_test.py @@ -13,7 +13,7 @@ from core.errors import AccountHasResourcesError, ForbiddenError, UserNotFoundError from database.users import UserGroup -from routers.openml.users import delete_user_account +from routers.users import delete_user_account from tests.users import ADMIN_USER, OWNER_USER, SOME_USER, ApiKey @@ -69,7 +69,7 @@ async def test_delete_user_api_success_self_delete( disposable_user: DisposableUser, mocker: pytest_mock.MockerFixture, ) -> None: - log_info = mocker.patch("routers.openml.users.logger.info") + log_info = mocker.patch("routers.users.logger.info") response = await py_api.delete( f"/users/{disposable_user.user_id}", @@ -185,7 +185,7 @@ async def test_delete_user_integrity_error_logs_and_raises_conflict( "DELETE FROM users", {"user_id": disposable_user.user_id}, Exception("fk") ), ) - log_error = mocker.patch("routers.openml.users.logger.error") + log_error = mocker.patch("routers.users.logger.error") with pytest.raises(AccountHasResourcesError, match="Cannot delete this account") as exc_info: await delete_user_account( From 9e87ea2af31b02172236c9ffcb0532ba9e38c2a1 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 16:49:27 +0200 Subject: [PATCH 04/10] Provide descriptions for public methods and modules --- src/routers/datasets.py | 29 +++++++++++++++++++++-------- src/routers/dependencies.py | 5 +++++ src/routers/estimation_procedure.py | 6 ++++++ src/routers/evaluations.py | 10 ++++++++++ src/routers/flows.py | 3 +++ src/routers/qualities.py | 8 ++++++++ src/routers/runs.py | 11 +++++------ src/routers/setups.py | 4 ++-- src/routers/study.py | 10 +++++++--- src/routers/tasks.py | 10 +++++++--- src/routers/tasktype.py | 8 ++++++++ src/routers/users.py | 2 +- src/schemas/core.py | 10 ++++++++++ 13 files changed, 93 insertions(+), 23 deletions(-) diff --git a/src/routers/datasets.py b/src/routers/datasets.py index b741f4d..9667ac1 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -1,9 +1,17 @@ +"""Defines endpoints relating to Datasets. + +A dataset includes both the "data", e.g., the table or parquet file, as well as its metadata. +The metadata is partially provided by the user (for example, the name or description), +'features' that are partially parsed from the data file (for example, column names), +and 'qualities' (or meta-features) that describe the data (for example, number of rows or columns). +""" + import asyncio import re from datetime import datetime from enum import StrEnum from http import HTTPStatus -from typing import TYPE_CHECKING, Annotated, Any, Literal, NamedTuple, NotRequired, TypedDict +from typing import TYPE_CHECKING, Annotated, Any, Literal, NamedTuple from fastapi import APIRouter, Body, Depends, Query from loguru import logger @@ -49,6 +57,7 @@ fetch_user_or_raise, userdb_connection, ) +from schemas.core import TagInfo from schemas.datasets.openml import DatasetMetadata, DatasetStatus, Feature, FeatureType if TYPE_CHECKING: @@ -67,6 +76,7 @@ async def tag_dataset( user: Annotated[User, Depends(fetch_user_or_raise)], expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[str, dict[str, Any]]: + """Add a tag to the dataset, this tag is publicly visible to all users.""" try: await database.datasets.tag(data_id, tag, user_id=user.user_id, connection=expdb_db) except ForeignKeyConstraintError: @@ -85,11 +95,6 @@ async def tag_dataset( } -class TagInfo(TypedDict): - id: str - tag: NotRequired[TagString | list[TagString]] - - @router.post(path="/untag", deprecated=True) async def untag_dataset_like_php( data_id: Annotated[Identifier, Body()], @@ -97,6 +102,7 @@ async def untag_dataset_like_php( user: Annotated[User, Depends(fetch_user_or_raise)], expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[Literal["data_untag"], TagInfo]: + """Remove a tag from the dataset with a response similar to the old PHP server.""" await untag_dataset(data_id, tag, user, expdb_db) tags = await database.datasets.get_tags_for(id_=data_id, connection=expdb_db) tag_info: TagInfo = {"id": str(data_id)} @@ -114,6 +120,7 @@ async def untag_dataset( user: Annotated[User, Depends(fetch_user_or_raise)], expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> None: + """Remove a tag that you added to a dataset, or from a dataset you uploaded.""" dataset_tag = await database.datasets.get_tag(identifier, tag, expdb_db) if not dataset_tag: try: @@ -130,6 +137,8 @@ async def untag_dataset( class DatasetStatusFilter(StrEnum): + """Legal filter values for the Dataset Status filter.""" + ACTIVE = DatasetStatus.ACTIVE DEACTIVATED = DatasetStatus.DEACTIVATED IN_PREPARATION = DatasetStatus.IN_PREPARATION @@ -182,6 +191,7 @@ async def list_datasets( # noqa: PLR0913, C901 status: Annotated[DatasetStatusFilter, Body()] = DatasetStatusFilter.ACTIVE, user: Annotated[User | None, Depends(fetch_user)] = None, ) -> list[dict[str, Any]]: + """List all datasets that match the filters.""" status_subquery = text( """ SELECT ds1.`did`, ds1.`status` @@ -308,6 +318,8 @@ async def list_datasets( # noqa: PLR0913, C901 class ProcessingInformation(NamedTuple): + """Metadata about an attempt to process a dataset.""" + date: datetime | None warning: str | None error: str | None @@ -358,6 +370,7 @@ async def get_dataset_features( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> list[Feature]: + """Return metadata for each feature (column) in the dataset.""" assert expdb is not None # noqa: S101 await _get_dataset_raise_otherwise(dataset_id, user, expdb) features, ontologies = await asyncio.gather( @@ -402,6 +415,7 @@ async def update_dataset_status( user: Annotated[User, Depends(fetch_user_or_raise)], expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[str, str | int]: + """Update the status of the dataset. Can be used to deactivate a dataset.""" dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb) can_deactivate = dataset.uploader == user.user_id or await user.is_admin() @@ -456,8 +470,7 @@ async def get_dataset( expdb_db: Annotated[AsyncConnection, Depends(expdb_connection)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> DatasetMetadata: - assert user_db is not None # noqa: S101 - assert expdb_db is not None # noqa: S101 + """Get the user-provided metadata for a dataset.""" dataset = await _get_dataset_raise_otherwise(dataset_id, user, expdb_db) if not ( dataset_file := await database.datasets.get_file( diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index 87ea538..cba78cd 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -1,3 +1,8 @@ +"""Common dependencies for router endpoints. + +See also https://fastapi.tiangolo.com/reference/dependencies/ +""" + from collections.abc import AsyncGenerator, AsyncIterator from typing import TYPE_CHECKING, Annotated diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index d8532dc..2af7510 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -1,3 +1,5 @@ +"""Defines endpoints relating to Estimation Procedures.""" + from typing import TYPE_CHECKING, Annotated from fastapi import APIRouter, Depends @@ -16,5 +18,9 @@ async def get_estimation_procedures( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> list[EstimationProcedure]: + """Return a list with descriptions of estimation procedures. + + Estimation procedures define how to evaluate a model, e.g., 5-repeated 2-fold cross-validation. + """ procedures = await database.evaluations.get_estimation_procedures(expdb) return list(procedures) diff --git a/src/routers/evaluations.py b/src/routers/evaluations.py index f289131..65cb450 100644 --- a/src/routers/evaluations.py +++ b/src/routers/evaluations.py @@ -1,3 +1,5 @@ +"""Defines endpoints relating to Evaluation Measures.""" + from typing import TYPE_CHECKING, Annotated from fastapi import APIRouter, Depends @@ -15,6 +17,14 @@ async def get_evaluation_measures( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> list[str]: + """Return a list of evaluation measure names. + + Evaluation measures are any function that is computed over either predictions + or may otherwise be provided as run metadata. Examples include "area_under_roc_curve", + "recall", or "usercpu_time_millis". This may be used for filtering or ordering evaluations. + + There are no detailed descriptions available for these measures at this time. + """ functions = await database.evaluations.get_math_functions( function_type="EvaluationFunction", connection=expdb, diff --git a/src/routers/flows.py b/src/routers/flows.py index 1df5f08..b80a839 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -1,3 +1,5 @@ +"""Defines endpoints relating to Flows.""" + import asyncio from typing import TYPE_CHECKING, Annotated, Literal @@ -39,6 +41,7 @@ async def get_flow( flow_id: Identifier, expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> Flow: + """Get a Flow by its identifier.""" flow = await database.flows.get(flow_id, expdb) if not flow: msg = f"Flow with id {flow_id} not found." diff --git a/src/routers/qualities.py b/src/routers/qualities.py index f863fc2..fa10e8a 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -1,3 +1,9 @@ +"""Defines endpoints relating to Qualities. + +Qualities are computed meta-features of datasets, such as the number of rows or columns. +Qualities are computed automatically by an evaluation engine, they are not provided by users. +""" + from typing import TYPE_CHECKING, Annotated, Literal from fastapi import APIRouter, Depends @@ -26,6 +32,7 @@ async def list_qualities( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[Literal["data_qualities_list"], dict[Literal["quality"], list[str]]]: + """List names of all computed qualities (dataset metafeatures).""" qualities = await database.qualities.list_all_qualities(connection=expdb) return { "data_qualities_list": { @@ -40,6 +47,7 @@ async def get_qualities( user: Annotated[User | None, Depends(fetch_user)], expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> list[Quality]: + """Get computed qualities (metafeatures) for a dataset.""" dataset = await database.datasets.get(dataset_id, expdb) if not dataset or not await _user_has_access(dataset, user): msg = f"Dataset with id {dataset_id} not found." diff --git a/src/routers/runs.py b/src/routers/runs.py index 53ba5fa..8484b5e 100644 --- a/src/routers/runs.py +++ b/src/routers/runs.py @@ -1,4 +1,4 @@ -"""Endpoints for run-related data.""" +"""Endpoints relating to Runs and Traces.""" import asyncio from dataclasses import dataclass @@ -149,11 +149,10 @@ async def get_run( userdb: Annotated[AsyncConnection, Depends(userdb_connection)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> Run: - """Get full metadata for a run by ID. - - No authentication or visibility check is performed — all runs are - publicly accessible. - """ + """Get full metadata for a run by ID.""" + # Authentication is not required because runs are always public. + # This mimics PHP API behavior, but is unintuitive for runs which have predictions + # made on private datasets. run = await database.runs.get(run_id, expdb) if run is None: msg = f"Run {run_id} not found." diff --git a/src/routers/setups.py b/src/routers/setups.py index 615f3fb..c91f3f6 100644 --- a/src/routers/setups.py +++ b/src/routers/setups.py @@ -1,4 +1,4 @@ -"""All endpoints that relate to setups.""" +"""Defines endpoints relating to Setups.""" from typing import TYPE_CHECKING, Annotated @@ -54,7 +54,7 @@ async def tag_setup( user: Annotated[User, Depends(fetch_user_or_raise)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[str, dict[str, str | list[str]]]: - """Add tag `tag` to setup with id `setup_id`.""" + """Add a tag to the setup, this tag is publicly visible to all users.""" try: await database.setups.tag(setup_id, tag, user.user_id, expdb_session) except ForeignKeyConstraintError: diff --git a/src/routers/study.py b/src/routers/study.py index 6cbdde8..df67aed 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -1,3 +1,5 @@ +"""Defines endpoints relating to studies.""" + from typing import TYPE_CHECKING, Annotated, Literal from fastapi import APIRouter, Body, Depends @@ -58,6 +60,8 @@ async def _get_study_raise_otherwise( class AttachDetachResponse(BaseModel): + """Response format for attaching or detaching an entity from a study.""" + study_id: int main_entity_type: StudyType @@ -69,7 +73,7 @@ async def attach_to_study( user: Annotated[User, Depends(fetch_user_or_raise)], expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> AttachDetachResponse: - assert expdb is not None # noqa: S101 + """Add runs or tasks to a study which is in preparation.""" if user is None: msg = "Authentication required." raise AuthenticationRequiredError(msg) @@ -123,7 +127,7 @@ async def create_study( user: Annotated[User, Depends(fetch_user_or_raise)], expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[Literal["study_id"], int]: - assert expdb is not None # noqa: S101 + """Create a new study.""" if study.main_entity_type == StudyType.RUN and study.tasks: msg = "Cannot create a run study with tasks." raise StudyInvalidTypeError(msg) @@ -160,7 +164,7 @@ async def get_study( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], user: Annotated[User | None, Depends(fetch_user)] = None, ) -> Study: - assert expdb is not None # noqa: S101 + """Get a study by id or alias.""" study = await _get_study_raise_otherwise(alias_or_id, user, expdb) study_data = await database.studies.get_study_data(study, expdb) return Study( diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 16353f0..91f9868 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -1,3 +1,5 @@ +"""Defines endpoints relating to Tasks.""" + import asyncio import json import re @@ -41,6 +43,7 @@ async def tag_task( user: Annotated[User, Depends(fetch_user_or_raise)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> dict[str, dict[str, Any]]: + """Add a tag to the task, this tag is publicly visible to all users.""" try: await database.tasks.tag(task_id, tag, user_id=user.user_id, session=expdb_session) except ForeignKeyConstraintError: @@ -59,7 +62,7 @@ async def tag_task( } -def convert_template_xml_to_json(xml_template: str) -> dict[str, JSON]: +def _convert_template_xml_to_json(xml_template: str) -> dict[str, JSON]: json_template = xmltodict.parse(xml_template.replace("oml:", "")) json_str = json.dumps(json_template) # To account for the differences between PHP and Python conversions: @@ -122,7 +125,7 @@ async def fill_template( ] } """ - json_template = convert_template_xml_to_json(template) + json_template = _convert_template_xml_to_json(template) return cast( "dict[str, JSON]", await _fill_json_template( @@ -453,6 +456,7 @@ async def get_task( expdb: Annotated[AsyncConnection, Depends(expdb_connection)], expdb_session: Annotated[AsyncSession, Depends(expdb_session)], ) -> Task: + """Get a task by identifier.""" if not (task := await database.tasks.get(task_id, expdb)): msg = f"Task {task_id} not found." raise TaskNotFoundError(msg) @@ -480,7 +484,7 @@ async def get_task( for (name, _), filled in zip(input_templates, filled_templates, strict=True) ] outputs = [ - convert_template_xml_to_json(template) | {"name": name} + _convert_template_xml_to_json(template) | {"name": name} for name, io, required, template in templates if io == "output" ] diff --git a/src/routers/tasktype.py b/src/routers/tasktype.py index 7cdba50..a92348a 100644 --- a/src/routers/tasktype.py +++ b/src/routers/tasktype.py @@ -1,3 +1,9 @@ +"""Defines endpoints for operations relating to task types. + +A task type is a prototype for a task that defines the expected input and output for it. +Examples include Supervised Classification, Supervised Regression, or Clustering. +""" + import json from typing import TYPE_CHECKING, Annotated, Any, Literal, cast @@ -35,6 +41,7 @@ async def list_task_types( Literal["task_types"], dict[Literal["task_type"], list[dict[str, str | None | list[Any]]]], ]: + """Return a high level description of all task types.""" task_types: list[dict[str, str | None | list[Any]]] = [ _normalize_task_type(ttype) for ttype in await get_task_types(expdb) ] @@ -46,6 +53,7 @@ async def get_task_type( task_type_id: int, expdb: Annotated[AsyncConnection, Depends(expdb_connection)], ) -> dict[Literal["task_type"], dict[str, str | None | list[str] | list[dict[str, str]]]]: + """Return a detailed description for the given task type, including expected inputs.""" task_type_record = await db_get_task_type(task_type_id, expdb) if task_type_record is None: msg = f"Task type {task_type_id} not found." diff --git a/src/routers/users.py b/src/routers/users.py index 43ea9bb..14a900e 100644 --- a/src/routers/users.py +++ b/src/routers/users.py @@ -1,4 +1,4 @@ -"""User account HTTP endpoints.""" +"""Defines endpoints relating to Users.""" from http import HTTPStatus from typing import Annotated diff --git a/src/schemas/core.py b/src/schemas/core.py index 77287de..b9bcb5e 100644 --- a/src/schemas/core.py +++ b/src/schemas/core.py @@ -1,6 +1,16 @@ from enum import StrEnum, auto +from typing import NotRequired, TypedDict + +from core.types import TagString class Visibility(StrEnum): PUBLIC = auto() PRIVATE = auto() + + +class TagInfo(TypedDict): + """Common PHP response format for tag operations.""" + + id: str + tag: NotRequired[TagString | list[TagString]] From cb09988f7a5d534013c732c31f2d4f934d242cc2 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 17:13:01 +0200 Subject: [PATCH 05/10] Rename file that manages database engines to avoid confusion --- src/database/{setup.py => engine.py} | 4 ++++ src/database/models/base.py | 2 +- src/main.py | 2 +- src/routers/dependencies.py | 2 +- tests/conftest.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) rename src/database/{setup.py => engine.py} (88%) diff --git a/src/database/setup.py b/src/database/engine.py similarity index 88% rename from src/database/setup.py rename to src/database/engine.py index cfd2306..80cf0c1 100644 --- a/src/database/setup.py +++ b/src/database/engine.py @@ -1,3 +1,5 @@ +"""Functions relating to the database engines.""" + import functools from loguru import logger @@ -27,11 +29,13 @@ def _create_engine(db_config: DatabaseConfiguration) -> AsyncEngine: @functools.cache def user_database() -> AsyncEngine: + """Get a database engine connected to the `openml` database.""" return _create_engine(get_config().openml_database) @functools.cache def expdb_database() -> AsyncEngine: + """Get a database engine connected to the `openml_expdb` database.""" return _create_engine(get_config().expdb_database) diff --git a/src/database/models/base.py b/src/database/models/base.py index 0a75055..9ded3e2 100644 --- a/src/database/models/base.py +++ b/src/database/models/base.py @@ -18,7 +18,7 @@ class ClassName(ExpDBReflected, Base): from sqlalchemy.ext.declarative import DeferredReflection from sqlalchemy.orm import DeclarativeBase -from database.setup import expdb_database, user_database +from database.engine import expdb_database, user_database UntypedRow = Row[Any] diff --git a/src/main.py b/src/main.py index 1d80e53..59e1cb8 100644 --- a/src/main.py +++ b/src/main.py @@ -26,8 +26,8 @@ request_response_logger, setup_log_sinks, ) +from database.engine import close_databases from database.models.base import reflect_db_schemas -from database.setup import close_databases from routers.datasets import router as datasets_router from routers.estimation_procedure import router as estimationprocedure_router from routers.evaluations import router as evaluationmeasures_router diff --git a/src/routers/dependencies.py b/src/routers/dependencies.py index cba78cd..f0706ef 100644 --- a/src/routers/dependencies.py +++ b/src/routers/dependencies.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from core.errors import AuthenticationFailedError, AuthenticationRequiredError -from database.setup import expdb_database, user_database +from database.engine import expdb_database, user_database from database.users import APIKey, User if TYPE_CHECKING: diff --git a/tests/conftest.py b/tests/conftest.py index 035af5c..ded9c54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,7 +22,7 @@ RoutingConfiguration, ) from core.types import Identifier -from database.setup import expdb_database, user_database +from database.engine import expdb_database, user_database from main import create_api from routers.dependencies import expdb_connection, userdb_connection from tests.users import OWNER_USER From 8a6cd4706419d7a2fe72752a5ff2f8b25e65bc15 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 7 Jul 2026 17:29:32 +0200 Subject: [PATCH 06/10] Remove the submodule for datasets because we only support 1 schema. --- src/core/access.py | 2 +- src/core/formatting.py | 2 +- src/database/datasets.py | 2 +- src/database/evaluations.py | 2 +- src/database/qualities.py | 2 +- src/routers/datasets.py | 2 +- src/routers/estimation_procedure.py | 2 +- src/routers/qualities.py | 2 +- src/routers/tasks.py | 2 +- .../{datasets/openml.py => datasets.py} | 26 ++++++++++++++++++- src/schemas/datasets/__init__.py | 9 ------- tests/routers/datasets_get_test.py | 2 +- tests/routers/datasets_status_test.py | 2 +- 13 files changed, 36 insertions(+), 21 deletions(-) rename src/schemas/{datasets/openml.py => datasets.py} (87%) delete mode 100644 src/schemas/datasets/__init__.py diff --git a/src/core/access.py b/src/core/access.py index 871fb48..3dc6189 100644 --- a/src/core/access.py +++ b/src/core/access.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any from database.users import User -from schemas.datasets.openml import Visibility +from schemas.datasets import Visibility if TYPE_CHECKING: from sqlalchemy.engine import Row diff --git a/src/core/formatting.py b/src/core/formatting.py index 832d9aa..a23324e 100644 --- a/src/core/formatting.py +++ b/src/core/formatting.py @@ -2,7 +2,7 @@ from config import get_config from database.models.base import UntypedRow -from schemas.datasets.openml import DatasetFileFormat +from schemas.datasets import DatasetFileFormat def _str_to_bool(string: str) -> bool: diff --git a/src/database/datasets.py b/src/database/datasets.py index 312986e..1d6bf05 100644 --- a/src/database/datasets.py +++ b/src/database/datasets.py @@ -15,7 +15,7 @@ ForeignKeyConstraintError, ) from database.models.base import UntypedRow -from schemas.datasets.openml import DatasetStatus, Feature +from schemas.datasets import DatasetStatus, Feature if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/evaluations.py b/src/database/evaluations.py index 4ac1f40..7301d56 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -5,7 +5,7 @@ from core.formatting import _str_to_bool from database.models.base import UntypedRow -from schemas.datasets.openml import EstimationProcedure +from schemas.datasets import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/database/qualities.py b/src/database/qualities.py index 81bc1d4..bb239eb 100644 --- a/src/database/qualities.py +++ b/src/database/qualities.py @@ -5,7 +5,7 @@ from sqlalchemy import text from core.types import Identifier -from schemas.datasets.openml import Quality +from schemas.datasets import Quality if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 9667ac1..43e1888 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -58,7 +58,7 @@ userdb_connection, ) from schemas.core import TagInfo -from schemas.datasets.openml import DatasetMetadata, DatasetStatus, Feature, FeatureType +from schemas.datasets import DatasetMetadata, DatasetStatus, Feature, FeatureType if TYPE_CHECKING: from sqlalchemy.engine import Row diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index 2af7510..a1db21b 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -6,7 +6,7 @@ import database.evaluations from routers.dependencies import expdb_connection -from schemas.datasets.openml import EstimationProcedure +from schemas.datasets import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/qualities.py b/src/routers/qualities.py index fa10e8a..7a95090 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -20,7 +20,7 @@ from core.types import Identifier from database.users import User from routers.dependencies import expdb_connection, fetch_user -from schemas.datasets.openml import Quality +from schemas.datasets import Quality if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 91f9868..1917904 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -26,7 +26,7 @@ from database.models.base import UntypedRow from database.users import User from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise -from schemas.datasets.openml import Task +from schemas.datasets import Task if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession diff --git a/src/schemas/datasets/openml.py b/src/schemas/datasets.py similarity index 87% rename from src/schemas/datasets/openml.py rename to src/schemas/datasets.py index 767cbb8..e3300a7 100644 --- a/src/schemas/datasets/openml.py +++ b/src/schemas/datasets.py @@ -1,3 +1,5 @@ +"""Defines schemas for API responses relating to Datasets.""" + from __future__ import annotations from datetime import UTC, datetime @@ -8,35 +10,51 @@ class DatasetFileFormat(StrEnum): + """Allowed file formats for data files.""" + ARFF = "arff" SPARSE_ARFF = "sparse_arff" PARQUET = "parquet" class Visibility(StrEnum): + """Possible visibility statuses.""" + PUBLIC = "public" PRIVATE = "private" class DatasetStatus(StrEnum): + """Possible dataset statuses. + + - In Preparation: any uploaded dataset issn't yet successfully processed. + - Active: any uploaded dataset that has successfully been processed. + - Deactivated: use of the dataset is discouraged. + """ + ACTIVE = "active" DEACTIVATED = "deactivated" - IN_PROCESSING = "in processing" IN_PREPARATION = "in_preparation" class Quality(BaseModel): + """A computed quality (meta-feature) of the dataset.""" + name: str value: float | None class FeatureType(StrEnum): + """The type of a feature (column) in the dataset.""" + NUMERIC = "numeric" NOMINAL = "nominal" STRING = "string" class Feature(BaseModel): + """Metadata about a feature (column) in the dataset.""" + index: int name: str data_type: FeatureType @@ -49,6 +67,8 @@ class Feature(BaseModel): class EstimationProcedure(BaseModel): + """Description of an evaluation protocol, e.g., cross-validation.""" + id_: int = Field(serialization_alias="id") task_type_id: int name: str @@ -60,6 +80,8 @@ class EstimationProcedure(BaseModel): class DatasetMetadata(BaseModel): + """Metadata for a dataset.""" + id_: int = Field(json_schema_extra={"example": 1}, alias="id") visibility: Visibility = Field(json_schema_extra={"example": Visibility.PUBLIC}) status: DatasetStatus = Field(json_schema_extra={"example": DatasetStatus.ACTIVE}) @@ -137,6 +159,8 @@ class DatasetMetadata(BaseModel): class Task(BaseModel): + """Metadata for a task.""" + id_: int = Field(serialization_alias="id", json_schema_extra={"example": 59}) name: str = Field( json_schema_extra={"example": "Task 59: mfeat-pixel (Supervised Classification)"}, diff --git a/src/schemas/datasets/__init__.py b/src/schemas/datasets/__init__.py deleted file mode 100644 index c34a2d8..0000000 --- a/src/schemas/datasets/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Dataset schemas for the OpenML server API.""" - -from enum import StrEnum - - -class DatasetSchema(StrEnum): - DCAT_AP = "dcat-ap" - OPENML = "openml" - MLDCAT_AP = "mldcat-ap" diff --git a/tests/routers/datasets_get_test.py b/tests/routers/datasets_get_test.py index bfede5e..cf38c0d 100644 --- a/tests/routers/datasets_get_test.py +++ b/tests/routers/datasets_get_test.py @@ -13,7 +13,7 @@ from core.errors import DatasetNoAccessError, DatasetNotFoundError from database.users import User from routers.datasets import get_dataset -from schemas.datasets.openml import DatasetMetadata +from schemas.datasets import DatasetMetadata from tests.users import ADMIN_USER, DATASET_130_OWNER, NO_USER, SOME_USER, ApiKey if TYPE_CHECKING: diff --git a/tests/routers/datasets_status_test.py b/tests/routers/datasets_status_test.py index bee25db..581465a 100644 --- a/tests/routers/datasets_status_test.py +++ b/tests/routers/datasets_status_test.py @@ -7,7 +7,7 @@ from core.errors import DatasetAdminOnlyError, DatasetNotOwnedError from routers.datasets import update_dataset_status -from schemas.datasets.openml import DatasetStatus +from schemas.datasets import DatasetStatus from tests import constants from tests.users import ADMIN_USER, SOME_USER From ffad5699c80d34167a2cae8ae1f907709c170097 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 8 Jul 2026 10:35:43 +0200 Subject: [PATCH 07/10] Move Tasks and Estimation Procedure to a separate submodule --- src/database/evaluations.py | 2 +- src/routers/estimation_procedure.py | 2 +- src/routers/tasks.py | 2 +- src/schemas/datasets.py | 28 ------------------------ src/schemas/tasks.py | 34 +++++++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 src/schemas/tasks.py diff --git a/src/database/evaluations.py b/src/database/evaluations.py index 7301d56..c046d48 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -5,7 +5,7 @@ from core.formatting import _str_to_bool from database.models.base import UntypedRow -from schemas.datasets import EstimationProcedure +from schemas.tasks import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/estimation_procedure.py b/src/routers/estimation_procedure.py index a1db21b..d7e76d2 100644 --- a/src/routers/estimation_procedure.py +++ b/src/routers/estimation_procedure.py @@ -6,7 +6,7 @@ import database.evaluations from routers.dependencies import expdb_connection -from schemas.datasets import EstimationProcedure +from schemas.tasks import EstimationProcedure if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection diff --git a/src/routers/tasks.py b/src/routers/tasks.py index 1917904..ae01a30 100644 --- a/src/routers/tasks.py +++ b/src/routers/tasks.py @@ -26,7 +26,7 @@ from database.models.base import UntypedRow from database.users import User from routers.dependencies import Pagination, expdb_connection, expdb_session, fetch_user_or_raise -from schemas.datasets import Task +from schemas.tasks import Task if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession diff --git a/src/schemas/datasets.py b/src/schemas/datasets.py index e3300a7..7307b28 100644 --- a/src/schemas/datasets.py +++ b/src/schemas/datasets.py @@ -4,7 +4,6 @@ from datetime import UTC, datetime from enum import StrEnum -from typing import Any from pydantic import BaseModel, Field, HttpUrl @@ -66,19 +65,6 @@ class Feature(BaseModel): nominal_values: list[str] | None -class EstimationProcedure(BaseModel): - """Description of an evaluation protocol, e.g., cross-validation.""" - - id_: int = Field(serialization_alias="id") - task_type_id: int - name: str - type_: str = Field(serialization_alias="type") - percentage: int | None - repeats: int | None - folds: int | None - stratified_sampling: bool | None - - class DatasetMetadata(BaseModel): """Metadata for a dataset.""" @@ -156,17 +142,3 @@ class DatasetMetadata(BaseModel): json_schema_extra={"example": "https://www.openml.org/d/2"}, ) md5_checksum: str = Field(json_schema_extra={"example": "d01f6ccd68c88b749b20bbe897de3713"}) - - -class Task(BaseModel): - """Metadata for a task.""" - - id_: int = Field(serialization_alias="id", json_schema_extra={"example": 59}) - name: str = Field( - json_schema_extra={"example": "Task 59: mfeat-pixel (Supervised Classification)"}, - ) - task_type_id: int = Field(json_schema_extra={"example": 1}) - task_type: str = Field(json_schema_extra={"example": "Supervised Classification"}) - input_: list[dict[str, Any]] = Field(serialization_alias="input") - output: list[dict[str, Any]] - tags: list[str] = Field(default_factory=list) diff --git a/src/schemas/tasks.py b/src/schemas/tasks.py new file mode 100644 index 0000000..475f1a4 --- /dev/null +++ b/src/schemas/tasks.py @@ -0,0 +1,34 @@ +"""Defines schemas for API responses relating to Tasks.""" + +from typing import Any + +from pydantic import BaseModel, Field + +from core.types import Identifier + + +class Task(BaseModel): + """Metadata for a task.""" + + id_: Identifier = Field(serialization_alias="id", json_schema_extra={"example": 59}) + name: str = Field( + json_schema_extra={"example": "Task 59: mfeat-pixel (Supervised Classification)"}, + ) + task_type_id: Identifier = Field(json_schema_extra={"example": 1}) + task_type: str = Field(json_schema_extra={"example": "Supervised Classification"}) + input_: list[dict[str, Any]] = Field(serialization_alias="input") + output: list[dict[str, Any]] + tags: list[str] = Field(default_factory=list) + + +class EstimationProcedure(BaseModel): + """Description of an evaluation protocol, e.g., cross-validation.""" + + id_: Identifier = Field(serialization_alias="id") + task_type_id: Identifier + name: str + type_: str = Field(serialization_alias="type") + percentage: int | None + repeats: int | None + folds: int | None + stratified_sampling: bool | None From 7fc8683fb3506819c65b2f789db9da223c1afcb6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 8 Jul 2026 10:43:08 +0200 Subject: [PATCH 08/10] Mark public function as public --- src/core/access.py | 2 +- src/core/conversions.py | 4 ++-- src/core/formatting.py | 6 +++--- src/database/evaluations.py | 4 ++-- src/routers/datasets.py | 22 +++++++++++----------- src/routers/flows.py | 4 ++-- src/routers/qualities.py | 4 ++-- src/routers/study.py | 6 +++--- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/core/access.py b/src/core/access.py index 3dc6189..920e920 100644 --- a/src/core/access.py +++ b/src/core/access.py @@ -7,7 +7,7 @@ from sqlalchemy.engine import Row -async def _user_has_access( +async def user_has_access( dataset: Row[Any], user: User | None = None, ) -> bool: diff --git a/src/core/conversions.py b/src/core/conversions.py index 5ac13c4..597b76a 100644 --- a/src/core/conversions.py +++ b/src/core/conversions.py @@ -3,7 +3,7 @@ from typing import Any -def _str_to_num(string: str) -> int | float | str: +def str_to_num(string: str) -> int | float | str: """Try to convert the string to integer, otherwise float, otherwise returns the input.""" if string.isdigit(): return int(string) @@ -23,7 +23,7 @@ def nested_str_to_num(obj: Any) -> Any: For dictionaries, only the values will be converted. """ if isinstance(obj, str): - return _str_to_num(obj) + return str_to_num(obj) if isinstance(obj, Mapping): return {key: nested_str_to_num(val) for key, val in obj.items()} if isinstance(obj, Iterable): diff --git a/src/core/formatting.py b/src/core/formatting.py index a23324e..a287c52 100644 --- a/src/core/formatting.py +++ b/src/core/formatting.py @@ -5,7 +5,7 @@ from schemas.datasets import DatasetFileFormat -def _str_to_bool(string: str) -> bool: +def str_to_bool(string: str) -> bool: if string.casefold() in ["true", "1", "yes", "y"]: return True if string.casefold() in ["false", "0", "no", "n"]: @@ -24,7 +24,7 @@ def _format_parquet_url(dataset: UntypedRow) -> str | None: return f"{minio_base_url}datasets/{ten_thousands_prefix}/{padded_id}/dataset_{dataset.did}.pq" -def _format_dataset_url(dataset: UntypedRow) -> str: +def format_dataset_url(dataset: UntypedRow) -> str: base_url = get_config().routing.server_url filename = f"{html.escape(dataset.name)}.{dataset.format.lower()}" return f"{base_url}data/v1/download/{dataset.file_id}/{filename}" @@ -37,7 +37,7 @@ def _safe_unquote(text: str | None) -> str | None: return text.strip("'\"") or None -def _csv_as_list(text: str | None, *, unquote_items: bool = True) -> list[str]: +def csv_as_list(text: str | None, *, unquote_items: bool = True) -> list[str]: """Return comma-separated values in `text` as list, optionally remove quotes.""" if not text: return [] diff --git a/src/database/evaluations.py b/src/database/evaluations.py index c046d48..ce0d230 100644 --- a/src/database/evaluations.py +++ b/src/database/evaluations.py @@ -3,7 +3,7 @@ from sqlalchemy import text -from core.formatting import _str_to_bool +from core.formatting import str_to_bool from database.models.base import UntypedRow from schemas.tasks import EstimationProcedure @@ -41,7 +41,7 @@ async def get_estimation_procedures(connection: AsyncConnection) -> list[Estimat rows = row.mappings().all() typed_rows = [ { - k: v if k != "stratified_sampling" or v is None else _str_to_bool(v) + k: v if k != "stratified_sampling" or v is None else str_to_bool(v) for k, v in row.items() } for row in rows diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 43e1888..5605d83 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -19,7 +19,7 @@ import database.datasets import database.qualities -from core.access import _user_has_access +from core.access import user_has_access from core.errors import ( DatasetAdminOnlyError, DatasetNoAccessError, @@ -37,9 +37,9 @@ TagNotOwnedError, ) from core.formatting import ( - _csv_as_list, - _format_dataset_url, _format_parquet_url, + csv_as_list, + format_dataset_url, ) from core.types import ( CasualString128, @@ -357,7 +357,7 @@ async def _get_dataset_raise_otherwise( msg = f"No dataset with id {dataset_id} found." raise DatasetNotFoundError(msg) - if not await _user_has_access(dataset=dataset, user=user): + if not await user_has_access(dataset=dataset, user=user): msg = f"No access granted to dataset {dataset_id}." raise DatasetNoAccessError(msg) @@ -492,15 +492,15 @@ async def get_dataset( if description: description_ = description.description.replace("\r", "").strip() - dataset_url = _format_dataset_url(dataset) + dataset_url = format_dataset_url(dataset) parquet_url = _format_parquet_url(dataset) - contributors = _csv_as_list(dataset.contributor, unquote_items=True) - creators = _csv_as_list(dataset.creator, unquote_items=True) - ignore_attribute = _csv_as_list(dataset.ignore_attribute, unquote_items=True) - row_id_attribute = _csv_as_list(dataset.row_id_attribute, unquote_items=True) - original_data_url = _csv_as_list(dataset.original_data_url, unquote_items=True) - default_target_attribute = _csv_as_list(dataset.default_target_attribute, unquote_items=True) + contributors = csv_as_list(dataset.contributor, unquote_items=True) + creators = csv_as_list(dataset.creator, unquote_items=True) + ignore_attribute = csv_as_list(dataset.ignore_attribute, unquote_items=True) + row_id_attribute = csv_as_list(dataset.row_id_attribute, unquote_items=True) + original_data_url = csv_as_list(dataset.original_data_url, unquote_items=True) + default_target_attribute = csv_as_list(dataset.default_target_attribute, unquote_items=True) return DatasetMetadata( id=dataset.did, diff --git a/src/routers/flows.py b/src/routers/flows.py index b80a839..0bb02cb 100644 --- a/src/routers/flows.py +++ b/src/routers/flows.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends import database.flows -from core.conversions import _str_to_num +from core.conversions import str_to_num from core.errors import FlowNotFoundError from core.types import Identifier from routers.dependencies import expdb_connection @@ -58,7 +58,7 @@ async def get_flow( # PHP sets the default value to [], not sure where that comes from. # In the modern interface, `None` is used instead for now, but I think it might # make more sense to omit it if there is none. - default_value=_str_to_num(parameter.default_value) if parameter.default_value else None, + default_value=str_to_num(parameter.default_value) if parameter.default_value else None, data_type=parameter.data_type, description=parameter.description, ) diff --git a/src/routers/qualities.py b/src/routers/qualities.py index 7a95090..58d08e6 100644 --- a/src/routers/qualities.py +++ b/src/routers/qualities.py @@ -10,7 +10,7 @@ import database.datasets import database.qualities -from core.access import _user_has_access +from core.access import user_has_access from core.errors import ( DatasetNotFoundError, DatasetNotProcessedError, @@ -49,7 +49,7 @@ async def get_qualities( ) -> list[Quality]: """Get computed qualities (metafeatures) for a dataset.""" dataset = await database.datasets.get(dataset_id, expdb) - if not dataset or not await _user_has_access(dataset, user): + if not dataset or not await user_has_access(dataset, user): msg = f"Dataset with id {dataset_id} not found." raise DatasetNotFoundError( msg, diff --git a/src/routers/study.py b/src/routers/study.py index df67aed..a686929 100644 --- a/src/routers/study.py +++ b/src/routers/study.py @@ -17,7 +17,7 @@ StudyNotFoundError, StudyPrivateError, ) -from core.formatting import _str_to_bool +from core.formatting import str_to_bool from core.types import Identifier from database.models.base import UntypedRow from database.users import User @@ -53,7 +53,7 @@ async def _get_study_raise_otherwise( if study.creator != user.user_id and not await user.is_admin(): msg = "Study is private." raise StudyPrivateError(msg) - if _str_to_bool(study.legacy): + if str_to_bool(study.legacy): msg = "Legacy studies are no longer supported." raise StudyLegacyError(msg) return study @@ -168,7 +168,7 @@ async def get_study( study = await _get_study_raise_otherwise(alias_or_id, user, expdb) study_data = await database.studies.get_study_data(study, expdb) return Study( - _legacy=_str_to_bool(study.legacy), + _legacy=str_to_bool(study.legacy), id_=study.id, name=study.name, alias=study.alias, From 5d8524a48700990a0b1cd4887a42bb611f0f5ba9 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 8 Jul 2026 11:19:03 +0200 Subject: [PATCH 09/10] [no ci] fix typo --- src/schemas/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schemas/datasets.py b/src/schemas/datasets.py index 7307b28..be4795e 100644 --- a/src/schemas/datasets.py +++ b/src/schemas/datasets.py @@ -26,7 +26,7 @@ class Visibility(StrEnum): class DatasetStatus(StrEnum): """Possible dataset statuses. - - In Preparation: any uploaded dataset issn't yet successfully processed. + - In Preparation: any uploaded dataset isn't yet successfully processed. - Active: any uploaded dataset that has successfully been processed. - Deactivated: use of the dataset is discouraged. """ From 537746008067fe3d11575dbbbea6f30a0e22565c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 8 Jul 2026 11:28:09 +0200 Subject: [PATCH 10/10] Move formatting function to only module that uses it --- src/core/formatting.py | 29 +---------------------------- src/routers/datasets.py | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/src/core/formatting.py b/src/core/formatting.py index a287c52..33146f4 100644 --- a/src/core/formatting.py +++ b/src/core/formatting.py @@ -1,8 +1,4 @@ -import html - -from config import get_config -from database.models.base import UntypedRow -from schemas.datasets import DatasetFileFormat +"""Utilities for formatting and converting text data.""" def str_to_bool(string: str) -> bool: @@ -14,29 +10,6 @@ def str_to_bool(string: str) -> bool: raise ValueError(msg) -def _format_parquet_url(dataset: UntypedRow) -> str | None: - if dataset.format.lower() != DatasetFileFormat.ARFF: - return None - - minio_base_url = get_config().routing.minio_url - ten_thousands_prefix = f"{dataset.did // 10_000:04d}" - padded_id = f"{dataset.did:04d}" - return f"{minio_base_url}datasets/{ten_thousands_prefix}/{padded_id}/dataset_{dataset.did}.pq" - - -def format_dataset_url(dataset: UntypedRow) -> str: - base_url = get_config().routing.server_url - filename = f"{html.escape(dataset.name)}.{dataset.format.lower()}" - return f"{base_url}data/v1/download/{dataset.file_id}/{filename}" - - -def _safe_unquote(text: str | None) -> str | None: - """Remove any open and closing quotes and return the remainder if non-empty.""" - if not text: - return None - return text.strip("'\"") or None - - def csv_as_list(text: str | None, *, unquote_items: bool = True) -> list[str]: """Return comma-separated values in `text` as list, optionally remove quotes.""" if not text: diff --git a/src/routers/datasets.py b/src/routers/datasets.py index 5605d83..2718e7b 100644 --- a/src/routers/datasets.py +++ b/src/routers/datasets.py @@ -7,6 +7,7 @@ """ import asyncio +import html import re from datetime import datetime from enum import StrEnum @@ -19,6 +20,7 @@ import database.datasets import database.qualities +from config import get_config from core.access import user_has_access from core.errors import ( DatasetAdminOnlyError, @@ -36,11 +38,7 @@ TagNotFoundError, TagNotOwnedError, ) -from core.formatting import ( - _format_parquet_url, - csv_as_list, - format_dataset_url, -) +from core.formatting import csv_as_list from core.types import ( CasualString128, Identifier, @@ -49,6 +47,7 @@ integer_range_regex, ) from database.exceptions import DuplicatePrimaryKeyError, ForeignKeyConstraintError +from database.models.base import UntypedRow from database.users import User from routers.dependencies import ( Pagination, @@ -58,7 +57,7 @@ userdb_connection, ) from schemas.core import TagInfo -from schemas.datasets import DatasetMetadata, DatasetStatus, Feature, FeatureType +from schemas.datasets import DatasetFileFormat, DatasetMetadata, DatasetStatus, Feature, FeatureType if TYPE_CHECKING: from sqlalchemy.engine import Row @@ -67,6 +66,22 @@ router = APIRouter(prefix="/datasets", tags=["datasets"]) +def _format_parquet_url(dataset: UntypedRow) -> str | None: + if dataset.format.lower() != DatasetFileFormat.ARFF: + return None + + minio_base_url = get_config().routing.minio_url + ten_thousands_prefix = f"{dataset.did // 10_000:04d}" + padded_id = f"{dataset.did:04d}" + return f"{minio_base_url}datasets/{ten_thousands_prefix}/{padded_id}/dataset_{dataset.did}.pq" + + +def _format_dataset_url(dataset: UntypedRow) -> str: + base_url = get_config().routing.server_url + filename = f"{html.escape(dataset.name)}.{dataset.format.lower()}" + return f"{base_url}data/v1/download/{dataset.file_id}/{filename}" + + @router.post( path="/tag", ) @@ -492,7 +507,7 @@ async def get_dataset( if description: description_ = description.description.replace("\r", "").strip() - dataset_url = format_dataset_url(dataset) + dataset_url = _format_dataset_url(dataset) parquet_url = _format_parquet_url(dataset) contributors = csv_as_list(dataset.contributor, unquote_items=True)