diff --git a/.github/workflows/testing-ci.yml b/.github/workflows/testing-ci.yml index a915d32..f006f5e 100644 --- a/.github/workflows/testing-ci.yml +++ b/.github/workflows/testing-ci.yml @@ -78,3 +78,40 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./.coverage.py312.xml fail_ci_if_error: false + + dlclive-compat: + name: DLCLive Compatibility • ${{ matrix.label }} • py${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.12'] + include: + - label: pypi-1.1 + dlclive_spec: deeplabcut-live==1.1 + - label: github-main + dlclive_spec: git+https://github.com/DeepLabCut/DeepLabCut-live.git@main + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + + - name: Install package + test dependencies + run: | + python -m pip install -U pip wheel + python -m pip install -e .[test] + + - name: Install matrix DLCLive build + run: | + python -m pip install --upgrade --force-reinstall "${{ matrix.dlclive_spec }}" + python -m pip show deeplabcut-live + + - name: Run DLCLive compatibility tests + run: | + python -m pytest -m dlclive_compat tests/compat/test_dlclive_package_compat.py -q diff --git a/dlclivegui/services/dlc_processor.py b/dlclivegui/services/dlc_processor.py index 42b5868..17dfae5 100644 --- a/dlclivegui/services/dlc_processor.py +++ b/dlclivegui/services/dlc_processor.py @@ -15,7 +15,7 @@ import numpy as np from PySide6.QtCore import QObject, Signal -from dlclivegui.config import DLCProcessorSettings +from dlclivegui.config import DLCProcessorSettings, ModelType from dlclivegui.processors.processor_utils import instantiate_from_scan from dlclivegui.temp import Engine # type: ignore # TODO use main package enum when released @@ -37,6 +37,66 @@ class PoseResult: pose: np.ndarray | None timestamp: float + packet: PosePacket | None = None + + +@dataclass(slots=True, frozen=True) +class PoseSource: + backend: str # e.g. "DLCLive" + model_type: ModelType | None = None + + +@dataclass(slots=True, frozen=True) +class PosePacket: + schema_version: int = 0 + keypoints: np.ndarray | None = None + keypoint_names: list[str] | None = None + individual_ids: list[str] | None = None + source: PoseSource = PoseSource(backend="DLCLive") + raw: Any | None = None + + +def validate_pose_array(pose: Any, *, source_backend: str = "DLCLive") -> np.ndarray: + """ + Validate pose output shape and dtype. + + Accepted runner output shapes: + - (K, 3): single-animal + - (N, K, 3): multi-animal + """ + try: + arr = np.asarray(pose) + except Exception as exc: + raise ValueError( + f"{source_backend} returned an invalid pose output format: could not convert to array ({exc})" + ) from exc + + if arr.ndim not in (2, 3): + raise ValueError( + f"{source_backend} returned an invalid pose output format:" + f" expected a 2D or 3D array, got ndim={arr.ndim}, shape={arr.shape!r}" + ) + + if arr.shape[-1] != 3: + raise ValueError( + f"{source_backend} returned an invalid pose output format:" + f" expected last dimension size 3 (x, y, likelihood), got shape={arr.shape!r}" + ) + + if arr.ndim == 2 and arr.shape[0] <= 0: + raise ValueError(f"{source_backend} returned an invalid pose output format: expected at least one keypoint") + if arr.ndim == 3 and (arr.shape[0] <= 0 or arr.shape[1] <= 0): + raise ValueError( + f"{source_backend} returned an invalid pose output format:" + f" expected at least one individual and one keypoint, got shape={arr.shape!r}" + ) + + if not np.issubdtype(arr.dtype, np.number): + raise ValueError( + f"{source_backend} returned an invalid pose output format: expected numeric values, got dtype={arr.dtype}" + ) + + return arr @dataclass @@ -269,8 +329,17 @@ def _process_frame( # Time GPU inference (and processor overhead when present) with self._timed_processor() as proc_holder: inference_start = time.perf_counter() - pose = self._dlc.get_pose(frame, frame_time=timestamp) + raw_pose: Any = self._dlc.get_pose(frame, frame_time=timestamp) inference_time = time.perf_counter() - inference_start + pose_arr: np.ndarray = validate_pose_array(raw_pose, source_backend="DLCLive") + pose_packet = PosePacket( + schema_version=0, + keypoints=pose_arr, + keypoint_names=None, + individual_ids=None, + source=PoseSource(backend="DLCLive", model_type=self._settings.model_type), + raw=raw_pose, + ) processor_overhead = 0.0 gpu_inference_time = inference_time @@ -280,7 +349,7 @@ def _process_frame( # Emit pose (measure signal overhead) signal_start = time.perf_counter() - self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp)) + self.pose_ready.emit(PoseResult(pose=pose_packet.keypoints, timestamp=timestamp, packet=pose_packet)) signal_time = time.perf_counter() - signal_start end_ts = time.perf_counter() diff --git a/pyproject.toml b/pyproject.toml index 02e8a96..f1e81ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ markers = [ "unit: Unit tests for individual components", "integration: Integration tests for component interaction", "functional: Functional tests for end-to-end workflows", + "dlclive_compat: Package/API compatibility tests against supported dlclive versions", "hardware: Tests that require specific hardware, notable camera backends", # "slow: Tests that take a long time to run", "gui: Tests that require GUI interaction", diff --git a/tests/compat/test_dlclive_package_compat.py b/tests/compat/test_dlclive_package_compat.py new file mode 100644 index 0000000..522cb8e --- /dev/null +++ b/tests/compat/test_dlclive_package_compat.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import importlib.metadata +import inspect +import os +from pathlib import Path + +import numpy as np +import pytest + + +def _get_signature_params(callable_obj) -> tuple[set[str], bool]: + """ + Return allowed keyword names for callable, allowing for **kwargs. + + Example: + >>> params, accepts_var_kw = _get_signature_params(lambda x, y, **kwargs: None, {"x", "y"}) + >>> params == {"x", "y"} + True + >>> accepts_var_kw + True + """ + sig = inspect.signature(callable_obj) + params = sig.parameters + accepts_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) + return params, accepts_var_kw + + +@pytest.mark.dlclive_compat +def test_dlclive_package_is_importable(): + from dlclive import DLCLive # noqa: PLC0415 + + assert DLCLive is not None + # Helpful for CI logs to confirm matrix install result. + _ = importlib.metadata.version("deeplabcut-live") + + +@pytest.mark.dlclive_compat +def test_dlclive_constructor_accepts_gui_expected_kwargs(): + """ + GUI passes these kwargs when constructing DLCLive. + This test catches upstream API changes that would break initialization. + """ + from dlclive import DLCLive # noqa: PLC0415 + + expected = { + "model_path", + "model_type", + "processor", + "dynamic", + "resize", + "precision", + "single_animal", + "device", + } + params, accepts_var_kw = _get_signature_params(DLCLive.__init__) + missing = {name for name in expected if name not in params} + assert not missing, f"DLCLive.__init__ is missing expected kwargs called by GUI: {sorted(missing)}" + assert accepts_var_kw, "DLCLive.__init__ should accept **kwargs" # captures current behavior + + +@pytest.mark.dlclive_compat +def test_dlclive_methods_match_gui_usage(): + """ + GUI expects: + - init_inference(frame) + - get_pose(frame, frame_time=) + """ + from dlclive import DLCLive # noqa: PLC0415 + + assert hasattr(DLCLive, "init_inference"), "DLCLive must provide init_inference(frame)" + assert hasattr(DLCLive, "get_pose"), "DLCLive must provide get_pose(frame, frame_time=...)" + + init_params, _ = _get_signature_params(DLCLive.init_inference) + init_missing = {name for name in {"frame"} if name not in init_params} + assert not init_missing, f"DLCLive.init_inference signature mismatch, missing: {sorted(init_missing)}" + + get_pose_params, _ = _get_signature_params(DLCLive.get_pose) + get_pose_missing = {name for name in {"frame", "frame_time"} if name not in get_pose_params} + assert not get_pose_missing, f"DLCLive.get_pose signature mismatch, missing: {sorted(get_pose_missing)}" + + +@pytest.mark.dlclive_compat +def test_dlclive_minimal_inference_smoke(): + """ + Real runtime smoke test (init + pose call) using a tiny exported model. + + Opt-in via env vars: + - DLCLIVE_TEST_MODEL_PATH: absolute/relative path to exported model folder/file + - DLCLIVE_TEST_MODEL_TYPE: optional model type (default: pytorch) + """ + model_path_env = os.getenv("DLCLIVE_TEST_MODEL_PATH", "").strip() + if not model_path_env: + pytest.skip("Set DLCLIVE_TEST_MODEL_PATH to run real DLCLive inference smoke test.") + + model_path = Path(model_path_env).expanduser() + if not model_path.exists(): + pytest.skip(f"DLCLIVE_TEST_MODEL_PATH does not exist: {model_path}") + + model_type = os.getenv("DLCLIVE_TEST_MODEL_TYPE", "pytorch").strip() or "pytorch" + + from dlclive import DLCLive # noqa: PLC0415 + + from dlclivegui.services.dlc_processor import validate_pose_array # noqa: PLC0415 + + dlc = DLCLive( + model_path=str(model_path), + model_type=model_type, + dynamic=[False, 0.5, 10], + resize=1.0, + precision="FP32", + single_animal=True, + ) + + frame = np.zeros((64, 64, 3), dtype=np.uint8) + dlc.init_inference(frame) + pose = dlc.get_pose(frame, frame_time=0.0) + pose_arr = validate_pose_array(pose, source_backend="DLCLive.get_pose") + + assert pose_arr.ndim in (2, 3) + assert pose_arr.shape[-1] == 3 + assert np.isfinite(pose_arr).all() diff --git a/tests/services/test_pose_contract.py b/tests/services/test_pose_contract.py new file mode 100644 index 0000000..2c909f7 --- /dev/null +++ b/tests/services/test_pose_contract.py @@ -0,0 +1,40 @@ +import numpy as np +import pytest + +from dlclivegui.services.dlc_processor import validate_pose_array + + +@pytest.mark.unit +def test_validate_pose_array_keeps_single_animal_shape(): + pose = np.ones((5, 3), dtype=np.float64) + out = validate_pose_array(pose) + assert out.shape == (5, 3) + assert out.dtype == np.float64 + + +@pytest.mark.unit +def test_validate_pose_array_accepts_multi_animal(): + pose = np.ones((2, 5, 3), dtype=np.float32) + out = validate_pose_array(pose) + assert out.shape == (2, 5, 3) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "bad_pose,expected", + [ + (np.ones((5, 2), dtype=np.float32), "last dimension size 3"), + (np.ones((2, 5, 4), dtype=np.float32), "last dimension size 3"), + (np.ones((3,), dtype=np.float32), "expected a 2D or 3D array"), + ], +) +def test_validate_pose_array_rejects_invalid_shapes(bad_pose, expected): + with pytest.raises(ValueError, match=expected): + validate_pose_array(bad_pose) + + +@pytest.mark.unit +def test_validate_pose_array_rejects_non_numeric(): + pose = np.array([[["x", "y", "p"]]], dtype=object) + with pytest.raises(ValueError, match="expected numeric values"): + validate_pose_array(pose) diff --git a/tox.ini b/tox.ini index d6f8d86..93a081d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,6 @@ description = Unit + smoke tests (exclude hardware) with coverage package = wheel extras = test - # Helpful defaults for headless CI runs (Qt/OpenCV): setenv = PYTHONWARNINGS = default @@ -21,9 +20,8 @@ setenv = OPENCV_VIDEOIO_PRIORITY_MSMF = 0 COVERAGE_FILE = {toxinidir}/.coverage.{envname} -# Keep behavior aligned with your GitHub Actions job: commands = - pytest -m "not hardware" --maxfail=1 --disable-warnings \ + pytest -m "not hardware and not dlclive_compat" --maxfail=1 --disable-warnings \ --cov={envsitepackagesdir}/dlclivegui \ --cov-report=xml:{toxinidir}/.coverage.{envname}.xml \ --cov-report=term-missing \