Reorganize

src/core/access.py

@@ -1,13 +1,13 @@
 from typing import TYPE_CHECKING, Any
 
 from database.users import User
-from schemas.datasets.openml import Visibility
+from schemas.datasets import Visibility
 
 if TYPE_CHECKING:
     from sqlalchemy.engine import Row
 
 
-async def _user_has_access(
+async def user_has_access(
     dataset: Row[Any],
     user: User | None = None,
 ) -> bool:

src/core/conversions.py

@@ -3,7 +3,7 @@
 from typing import Any
 
 
-def _str_to_num(string: str) -> int | float | str:
+def str_to_num(string: str) -> int | float | str:
     """Try to convert the string to integer, otherwise float, otherwise returns the input."""
     if string.isdigit():
         return int(string)
@@ -23,7 +23,7 @@ def nested_str_to_num(obj: Any) -> Any:
     For dictionaries, only the values will be converted.
     """
     if isinstance(obj, str):
-        return _str_to_num(obj)
+        return str_to_num(obj)
     if isinstance(obj, Mapping):
         return {key: nested_str_to_num(val) for key, val in obj.items()}
     if isinstance(obj, Iterable):

src/core/formatting.py

@@ -1,11 +1,7 @@
-import html
+"""Utilities for formatting and converting text data."""
 
-from config import get_config
-from database.schema.base import UntypedRow
-from schemas.datasets.openml import DatasetFileFormat
 
-
-def _str_to_bool(string: str) -> bool:
+def str_to_bool(string: str) -> bool:
     if string.casefold() in ["true", "1", "yes", "y"]:
         return True
     if string.casefold() in ["false", "0", "no", "n"]:
@@ -14,30 +10,7 @@ def _str_to_bool(string: str) -> bool:
     raise ValueError(msg)
 
 
-def _format_parquet_url(dataset: UntypedRow) -> str | None:
-    if dataset.format.lower() != DatasetFileFormat.ARFF:
-        return None
-
-    minio_base_url = get_config().routing.minio_url
-    ten_thousands_prefix = f"{dataset.did // 10_000:04d}"
-    padded_id = f"{dataset.did:04d}"
-    return f"{minio_base_url}datasets/{ten_thousands_prefix}/{padded_id}/dataset_{dataset.did}.pq"
-
-
-def _format_dataset_url(dataset: UntypedRow) -> str:
-    base_url = get_config().routing.server_url
-    filename = f"{html.escape(dataset.name)}.{dataset.format.lower()}"
-    return f"{base_url}data/v1/download/{dataset.file_id}/{filename}"
-
-
-def _safe_unquote(text: str | None) -> str | None:
-    """Remove any open and closing quotes and return the remainder if non-empty."""
-    if not text:
-        return None
-    return text.strip("'\"") or None
-
-
-def _csv_as_list(text: str | None, *, unquote_items: bool = True) -> list[str]:
+def csv_as_list(text: str | None, *, unquote_items: bool = True) -> list[str]:
     """Return comma-separated values in `text` as list, optionally remove quotes."""
     if not text:
         return []

src/routers/types.py → src/core/types.py

@@ -1,3 +1,5 @@
+"""Common types used in the definition of schemas and models."""
+
 from typing import Annotated
 
 from pydantic import Field

src/database/datasets.py

@@ -7,15 +7,15 @@
 from sqlalchemy import text
 from sqlalchemy.exc import IntegrityError
 
+from core.types import Identifier, TagString
 from database.exceptions import (
     _DUPLICATE_ENTRY,
     _FOREIGN_KEY_CONSTRAINT_FAILED,
     DuplicatePrimaryKeyError,
     ForeignKeyConstraintError,
 )
-from database.schema.base import UntypedRow
-from routers.types import Identifier, TagString
-from schemas.datasets.openml import DatasetStatus, Feature
+from database.models.base import UntypedRow
+from schemas.datasets import DatasetStatus, Feature
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection

src/database/setup.py → src/database/engine.py

@@ -1,3 +1,5 @@
+"""Functions relating to the database engines."""
+
 import functools
 
 from loguru import logger
@@ -27,11 +29,13 @@ def _create_engine(db_config: DatabaseConfiguration) -> AsyncEngine:
 
 @functools.cache
 def user_database() -> AsyncEngine:
+    """Get a database engine connected to the `openml` database."""
     return _create_engine(get_config().openml_database)
 
 
 @functools.cache
 def expdb_database() -> AsyncEngine:
+    """Get a database engine connected to the `openml_expdb` database."""
     return _create_engine(get_config().expdb_database)
 
 

src/database/evaluations.py

@@ -3,9 +3,9 @@
 
 from sqlalchemy import text
 
-from core.formatting import _str_to_bool
-from database.schema.base import UntypedRow
-from schemas.datasets.openml import EstimationProcedure
+from core.formatting import str_to_bool
+from database.models.base import UntypedRow
+from schemas.tasks import EstimationProcedure
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection
@@ -41,7 +41,7 @@ async def get_estimation_procedures(connection: AsyncConnection) -> list[Estimat
     rows = row.mappings().all()
     typed_rows = [
         {
-            k: v if k != "stratified_sampling" or v is None else _str_to_bool(v)
+            k: v if k != "stratified_sampling" or v is None else str_to_bool(v)
             for k, v in row.items()
         }
         for row in rows

src/database/flows.py

@@ -3,8 +3,8 @@
 
 from sqlalchemy import text
 
-from database.schema.base import UntypedRow
-from routers.types import Identifier
+from core.types import Identifier
+from database.models.base import UntypedRow
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection

src/database/schema/base.py → src/database/models/base.py

@@ -18,7 +18,7 @@ class ClassName(ExpDBReflected, Base):
 from sqlalchemy.ext.declarative import DeferredReflection
 from sqlalchemy.orm import DeclarativeBase
 
-from database.setup import expdb_database, user_database
+from database.engine import expdb_database, user_database
 
 UntypedRow = Row[Any]
 

src/database/schema/setups.py → src/database/models/setups.py

@@ -2,8 +2,8 @@
 
 from sqlalchemy.orm import Mapped, mapped_column
 
-from database.schema.base import Base, ExpDBReflected
-from routers.types import Identifier
+from core.types import Identifier
+from database.models.base import Base, ExpDBReflected
 
 
 class Setup(ExpDBReflected, Base):

src/database/schema/tags.py → src/database/models/tags.py

@@ -5,8 +5,8 @@
 from sqlalchemy import FetchedValue
 from sqlalchemy.orm import Mapped, mapped_column
 
-from database.schema.base import Base, ExpDBReflected
-from routers.types import Identifier, TagString
+from core.types import Identifier, TagString
+from database.models.base import Base, ExpDBReflected
 
 
 class Tag:

src/database/qualities.py

@@ -4,8 +4,8 @@
 
 from sqlalchemy import text
 
-from routers.types import Identifier
-from schemas.datasets.openml import Quality
+from core.types import Identifier
+from schemas.datasets import Quality
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection

src/database/runs.py

@@ -5,8 +5,8 @@
 
 from sqlalchemy import bindparam, text
 
-from database.schema.base import UntypedRow
-from routers.types import Identifier
+from core.types import Identifier
+from database.models.base import UntypedRow
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection

src/database/setups.py

@@ -6,15 +6,15 @@
 from sqlalchemy import select, text
 from sqlalchemy.exc import IntegrityError
 
+from core.types import Identifier, TagString
 from database.exceptions import (
     _DUPLICATE_ENTRY,
     _FOREIGN_KEY_CONSTRAINT_FAILED,
     DuplicatePrimaryKeyError,
     ForeignKeyConstraintError,
 )
-from database.schema.setups import Setup
-from database.schema.tags import SetupTag
-from routers.types import Identifier, TagString
+from database.models.setups import Setup
+from database.models.tags import SetupTag
 
 if TYPE_CHECKING:
     from sqlalchemy.engine import RowMapping

src/database/studies.py

@@ -5,9 +5,9 @@
 
 from sqlalchemy import text
 
-from database.schema.base import UntypedRow
+from core.types import Identifier
+from database.models.base import UntypedRow
 from database.users import User
-from routers.types import Identifier
 from schemas.study import CreateStudy, StudyType
 
 if TYPE_CHECKING:

src/database/tasks.py

@@ -4,15 +4,15 @@
 from sqlalchemy import select, text
 from sqlalchemy.exc import IntegrityError
 
+from core.types import Identifier, TagString
 from database.exceptions import (
     _DUPLICATE_ENTRY,
     _FOREIGN_KEY_CONSTRAINT_FAILED,
     DuplicatePrimaryKeyError,
     ForeignKeyConstraintError,
 )
-from database.schema.base import UntypedRow
-from database.schema.tags import TaskTag
-from routers.types import Identifier, TagString
+from database.models.base import UntypedRow
+from database.models.tags import TaskTag
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession

src/database/users.py

@@ -8,7 +8,7 @@
 from sqlalchemy import text
 
 from config import get_config
-from routers.types import Identifier
+from core.types import Identifier
 
 if TYPE_CHECKING:
     from sqlalchemy.ext.asyncio import AsyncConnection

src/main.py

@@ -26,19 +26,19 @@
     request_response_logger,
     setup_log_sinks,
 )
-from database.schema.base import reflect_db_schemas
-from database.setup import close_databases
-from routers.openml.datasets import router as datasets_router
-from routers.openml.estimation_procedure import router as estimationprocedure_router
-from routers.openml.evaluations import router as evaluationmeasures_router
-from routers.openml.flows import router as flows_router
-from routers.openml.qualities import router as qualities_router
-from routers.openml.runs import router as run_router
-from routers.openml.setups import router as setup_router
-from routers.openml.study import router as study_router
-from routers.openml.tasks import router as task_router
-from routers.openml.tasktype import router as ttype_router
-from routers.openml.users import router as users_router
+from database.engine import close_databases
+from database.models.base import reflect_db_schemas
+from routers.datasets import router as datasets_router
+from routers.estimation_procedure import router as estimationprocedure_router
+from routers.evaluations import router as evaluationmeasures_router
+from routers.flows import router as flows_router
+from routers.qualities import router as qualities_router
+from routers.runs import router as run_router
+from routers.setups import router as setup_router
+from routers.study import router as study_router
+from routers.tasks import router as task_router
+from routers.tasktype import router as ttype_router
+from routers.users import router as users_router
 
 
 @asynccontextmanager

src/routers/__init__.py

@@ -1 +1 @@
-"""API routers for the OpenML server API."""
+"""Routers for the OpenML API."""