From 57ea97658d791fad0e11fc9d8f5b1285c7af8a07 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 21:48:26 -0400 Subject: [PATCH 01/23] Migrate build system from setuptools to meson-python --- .github/codecov.yml | 2 +- .github/workflows/ci-i386.yml | 8 +- .github/workflows/ci.yaml | 5 +- .github/workflows/wheel.yaml | 2 +- .gitignore | 16 +- MANIFEST.in | 4 - docs/contributing.rst | 128 +++- docs/design/meson-python-migration.md | 656 ++++++++++++++++++ docs/index.rst | 25 +- docs/release.rst | 33 + meson.build | 20 + meson.options | 10 + numcodecs/tests/__init__.py | 3 - pyproject.toml | 67 +- setup.py | 385 ---------- {numcodecs => src/numcodecs}/__init__.py | 4 +- {numcodecs => src/numcodecs}/_shuffle.pyx | 0 {numcodecs => src/numcodecs}/_utils.pxd | 0 {numcodecs => src/numcodecs}/abc.py | 0 {numcodecs => src/numcodecs}/astype.py | 0 {numcodecs => src/numcodecs}/base64.py | 0 {numcodecs => src/numcodecs}/bitround.py | 0 {numcodecs => src/numcodecs}/blosc.pyx | 0 {numcodecs => src/numcodecs}/bz2.py | 0 {numcodecs => src/numcodecs}/categorize.py | 0 {numcodecs => src/numcodecs}/checksum32.py | 0 {numcodecs => src/numcodecs}/compat.py | 0 {numcodecs => src/numcodecs}/compat_ext.pxd | 0 {numcodecs => src/numcodecs}/compat_ext.pyx | 0 {numcodecs => src/numcodecs}/delta.py | 0 {numcodecs => src/numcodecs}/errors.py | 0 .../numcodecs}/fixedscaleoffset.py | 0 {numcodecs => src/numcodecs}/fletcher32.pyx | 0 {numcodecs => src/numcodecs}/gzip.py | 0 {numcodecs => src/numcodecs}/jenkins.pyx | 0 {numcodecs => src/numcodecs}/json.py | 0 {numcodecs => src/numcodecs}/lz4.pyx | 0 {numcodecs => src/numcodecs}/lzma.py | 0 src/numcodecs/meson.build | 290 ++++++++ {numcodecs => src/numcodecs}/msgpacks.py | 0 {numcodecs => src/numcodecs}/ndarray_like.py | 0 {numcodecs => src/numcodecs}/packbits.py | 0 {numcodecs => src/numcodecs}/pcodec.py | 3 +- {numcodecs => src/numcodecs}/pickles.py | 0 {numcodecs => src/numcodecs}/quantize.py | 0 {numcodecs => src/numcodecs}/registry.py | 0 {numcodecs => src/numcodecs}/shuffle.py | 0 src/numcodecs/version.py | 7 + {numcodecs => src/numcodecs}/vlen.pyx | 0 {numcodecs => src/numcodecs}/zarr3.py | 0 {numcodecs => src/numcodecs}/zfpy.py | 0 {numcodecs => src/numcodecs}/zlib.py | 0 {numcodecs => src/numcodecs}/zstd.pyx | 0 tests/__init__.py | 3 + {numcodecs/tests => tests}/common.py | 0 .../entry_points.txt | 0 .../package_with_entrypoint/__init__.py | 0 {numcodecs/tests => tests}/test_astype.py | 2 +- {numcodecs/tests => tests}/test_base64.py | 2 +- {numcodecs/tests => tests}/test_bitround.py | 0 {numcodecs/tests => tests}/test_blosc.py | 5 +- {numcodecs/tests => tests}/test_bz2.py | 2 +- {numcodecs/tests => tests}/test_categorize.py | 2 +- {numcodecs/tests => tests}/test_checksum32.py | 2 +- {numcodecs/tests => tests}/test_compat.py | 0 {numcodecs/tests => tests}/test_delta.py | 2 +- .../tests => tests}/test_entrypoints.py | 0 .../test_entrypoints_backport.py | 0 .../tests => tests}/test_fixedscaleoffset.py | 2 +- {numcodecs/tests => tests}/test_fletcher32.py | 1 - {numcodecs/tests => tests}/test_gzip.py | 2 +- {numcodecs/tests => tests}/test_jenkins.py | 2 +- {numcodecs/tests => tests}/test_json.py | 2 +- {numcodecs/tests => tests}/test_lz4.py | 2 +- {numcodecs/tests => tests}/test_lzma.py | 2 +- {numcodecs/tests => tests}/test_msgpacks.py | 2 +- .../tests => tests}/test_ndarray_like.py | 0 {numcodecs/tests => tests}/test_packbits.py | 2 +- {numcodecs/tests => tests}/test_pcodec.py | 2 +- {numcodecs/tests => tests}/test_pickles.py | 2 +- {numcodecs/tests => tests}/test_pyzstd.py | 1 - {numcodecs/tests => tests}/test_quantize.py | 2 +- {numcodecs/tests => tests}/test_registry.py | 0 {numcodecs/tests => tests}/test_shuffle.py | 2 +- {numcodecs/tests => tests}/test_vlen_array.py | 2 +- {numcodecs/tests => tests}/test_vlen_bytes.py | 2 +- {numcodecs/tests => tests}/test_vlen_utf8.py | 2 +- {numcodecs/tests => tests}/test_zarr3.py | 0 .../tests => tests}/test_zarr3_import.py | 0 {numcodecs/tests => tests}/test_zfpy.py | 2 +- {numcodecs/tests => tests}/test_zlib.py | 4 +- {numcodecs/tests => tests}/test_zstd.py | 4 +- 92 files changed, 1207 insertions(+), 523 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 docs/design/meson-python-migration.md create mode 100644 meson.build create mode 100644 meson.options delete mode 100644 numcodecs/tests/__init__.py delete mode 100644 setup.py rename {numcodecs => src/numcodecs}/__init__.py (97%) rename {numcodecs => src/numcodecs}/_shuffle.pyx (100%) rename {numcodecs => src/numcodecs}/_utils.pxd (100%) rename {numcodecs => src/numcodecs}/abc.py (100%) rename {numcodecs => src/numcodecs}/astype.py (100%) rename {numcodecs => src/numcodecs}/base64.py (100%) rename {numcodecs => src/numcodecs}/bitround.py (100%) rename {numcodecs => src/numcodecs}/blosc.pyx (100%) rename {numcodecs => src/numcodecs}/bz2.py (100%) rename {numcodecs => src/numcodecs}/categorize.py (100%) rename {numcodecs => src/numcodecs}/checksum32.py (100%) rename {numcodecs => src/numcodecs}/compat.py (100%) rename {numcodecs => src/numcodecs}/compat_ext.pxd (100%) rename {numcodecs => src/numcodecs}/compat_ext.pyx (100%) rename {numcodecs => src/numcodecs}/delta.py (100%) rename {numcodecs => src/numcodecs}/errors.py (100%) rename {numcodecs => src/numcodecs}/fixedscaleoffset.py (100%) rename {numcodecs => src/numcodecs}/fletcher32.pyx (100%) rename {numcodecs => src/numcodecs}/gzip.py (100%) rename {numcodecs => src/numcodecs}/jenkins.pyx (100%) rename {numcodecs => src/numcodecs}/json.py (100%) rename {numcodecs => src/numcodecs}/lz4.pyx (100%) rename {numcodecs => src/numcodecs}/lzma.py (100%) create mode 100644 src/numcodecs/meson.build rename {numcodecs => src/numcodecs}/msgpacks.py (100%) rename {numcodecs => src/numcodecs}/ndarray_like.py (100%) rename {numcodecs => src/numcodecs}/packbits.py (100%) rename {numcodecs => src/numcodecs}/pcodec.py (99%) rename {numcodecs => src/numcodecs}/pickles.py (100%) rename {numcodecs => src/numcodecs}/quantize.py (100%) rename {numcodecs => src/numcodecs}/registry.py (100%) rename {numcodecs => src/numcodecs}/shuffle.py (100%) create mode 100644 src/numcodecs/version.py rename {numcodecs => src/numcodecs}/vlen.pyx (100%) rename {numcodecs => src/numcodecs}/zarr3.py (100%) rename {numcodecs => src/numcodecs}/zfpy.py (100%) rename {numcodecs => src/numcodecs}/zlib.py (100%) rename {numcodecs => src/numcodecs}/zstd.pyx (100%) create mode 100644 tests/__init__.py rename {numcodecs/tests => tests}/common.py (100%) rename {numcodecs/tests => tests}/package_with_entrypoint-0.1.dist-info/entry_points.txt (100%) rename {numcodecs/tests => tests}/package_with_entrypoint/__init__.py (100%) rename {numcodecs/tests => tests}/test_astype.py (98%) rename {numcodecs/tests => tests}/test_base64.py (98%) rename {numcodecs/tests => tests}/test_bitround.py (100%) rename {numcodecs/tests => tests}/test_blosc.py (99%) rename {numcodecs/tests => tests}/test_bz2.py (98%) rename {numcodecs/tests => tests}/test_categorize.py (98%) rename {numcodecs/tests => tests}/test_checksum32.py (99%) rename {numcodecs/tests => tests}/test_compat.py (100%) rename {numcodecs/tests => tests}/test_delta.py (97%) rename {numcodecs/tests => tests}/test_entrypoints.py (100%) rename {numcodecs/tests => tests}/test_entrypoints_backport.py (100%) rename {numcodecs/tests => tests}/test_fixedscaleoffset.py (98%) rename {numcodecs/tests => tests}/test_fletcher32.py (99%) rename {numcodecs/tests => tests}/test_gzip.py (98%) rename {numcodecs/tests => tests}/test_jenkins.py (100%) rename {numcodecs/tests => tests}/test_json.py (98%) rename {numcodecs/tests => tests}/test_lz4.py (98%) rename {numcodecs/tests => tests}/test_lzma.py (98%) rename {numcodecs/tests => tests}/test_msgpacks.py (99%) rename {numcodecs/tests => tests}/test_ndarray_like.py (100%) rename {numcodecs/tests => tests}/test_packbits.py (96%) rename {numcodecs/tests => tests}/test_pcodec.py (98%) rename {numcodecs/tests => tests}/test_pickles.py (97%) rename {numcodecs/tests => tests}/test_pyzstd.py (99%) rename {numcodecs/tests => tests}/test_quantize.py (98%) rename {numcodecs/tests => tests}/test_registry.py (100%) rename {numcodecs/tests => tests}/test_shuffle.py (99%) rename {numcodecs/tests => tests}/test_vlen_array.py (98%) rename {numcodecs/tests => tests}/test_vlen_bytes.py (98%) rename {numcodecs/tests => tests}/test_vlen_utf8.py (98%) rename {numcodecs/tests => tests}/test_zarr3.py (100%) rename {numcodecs/tests => tests}/test_zarr3_import.py (100%) rename {numcodecs/tests => tests}/test_zfpy.py (98%) rename {numcodecs/tests => tests}/test_zlib.py (98%) rename {numcodecs/tests => tests}/test_zstd.py (99%) diff --git a/.github/codecov.yml b/.github/codecov.yml index e9b2ef7cd..c910cfa5e 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -8,7 +8,7 @@ coverage: default: target: 100 ignore: - - "numcodecs/tests/**" + - "tests/**" comment: layout: "diff, files" behavior: default diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 61f46b7c7..317a5c4d2 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -61,14 +61,8 @@ jobs: - name: Install numcodecs run: | - export DISABLE_NUMCODECS_AVX2="" uv venv - # TODO: Remove this conditional when pcodec supports Python 3.14 - if [[ "${{ matrix.python-version }}" == "3.14" ]]; then - uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] - else - uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] - fi + uv pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" shell: alpine.sh {0} - name: List installed packages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4030e1948..c65c45aa9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,12 +49,11 @@ jobs: - name: Install numcodecs run: | - export DISABLE_NUMCODECS_AVX2="" # TODO: Remove this conditional when pcodec supports Python 3.14 if [[ "${{ matrix.python-version }}" == "3.14" ]]; then - python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]" else - python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" fi - name: List installed packages diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index a006d513d..25e606e21 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -25,7 +25,7 @@ jobs: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x" - # note: CIBW_ENVIRONMENT is now set in pyproject.toml + # note: cibuildwheel config-settings are set in pyproject.toml steps: - uses: actions/checkout@v5 diff --git a/.gitignore b/.gitignore index 5e08553b3..6966ecfba 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,8 @@ __pycache__/ *$py.class # C extensions -numcodecs/**/*.c -numcodecs/**/*.h +src/numcodecs/**/*.c +src/numcodecs/**/*.h *.so # editor @@ -51,7 +51,7 @@ coverage.xml cover/ # Cython annotation files -numcodecs/*.html +src/numcodecs/*.html # Translations *.mo @@ -99,12 +99,14 @@ ENV/ # PyCharm .idea -# setuptools-scm -numcodecs/version.py +# Meson build artifacts +builddir/ +subprojects/ # Cython generated -numcodecs/*.c +src/numcodecs/*.c # pixi environments .pixi/* *.egg-info -pixi.lock \ No newline at end of file +pixi.lock +uv.lock \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 41513ecbc..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -recursive-include c-blosc * -recursive-include numcodecs *.pyx -recursive-include numcodecs *.pxd -include numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt diff --git a/docs/contributing.rst b/docs/contributing.rst index 166fa826c..f5ddc2f89 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -39,12 +39,7 @@ a bug report: e.g., "installed via pip into a virtual environment", or "installed using conda". Information about other packages installed can be obtained by executing ``pip list`` (if using pip to install packages) or ``conda list`` (if using conda to install - packages) from the operating system command prompt. The version of the Python - interpreter can be obtained by running a Python interactive session, e.g.:: - - $ python - Python 3.8.15 | packaged by conda-forge | (default, Nov 22 2022, 08:49:06) - [Clang 14.0.6 ] on darwin + packages) from the operating system command prompt. 3. An explanation of why the current behaviour is wrong/not desired, and what you expect instead. @@ -86,29 +81,95 @@ Then ``cd`` into the clone and add the ``upstream`` remote:: $ cd numcodecs $ git remote add upstream https://github.com/zarr-developers/numcodecs.git +Note the ``--recursive`` flag is required to clone the ``c-blosc`` git submodule. If you +forgot it, you can initialize it later with:: + + $ git submodule update --init --recursive + Creating a development environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To work with the NumCodecs source code, it is recommended to set up a Python virtual -environment and install all NumCodecs dependencies using the same versions as are used by -the core developers and continuous integration services. Assuming you have a Python -3 interpreter already installed matching the ``requires-python`` constraint from -``pyproject.toml``, and you have cloned the NumCodecs source code and your -current working directory is the root of the repository, you can do something -like the following:: +NumCodecs contains C and Cython extensions, so you need a C compiler and build +tooling in addition to Python. - $ python3 -m venv ~/pyenv/numcodecs-dev - $ source ~/pyenv/numcodecs-dev/bin/activate - $ pip install -e .[docs,test,msgpack,zfpy] +Prerequisites +""""""""""""" -You may need to initialize the submodule for c-blosc: +You need a C compiler available on your ``PATH``. - $ git submodule update --init --recursive +On Debian/Ubuntu:: + + $ sudo apt install build-essential + +On macOS (Xcode command line tools):: + + $ xcode-select --install + +On Windows, install `Visual Studio Build Tools +`_ with the "Desktop development +with C++" workload. + +Setting up with uv (recommended) +""""""""""""""""""""""""""""""""" + +`uv `_ is a fast Python package manager. First, +bootstrap the build dependencies, then sync the project:: + + $ uv venv + $ uv pip install meson-python meson ninja cython numpy setuptools-scm + $ uv sync --group dev --extra test --extra msgpack + +The first ``uv pip install`` step bootstraps the build tools into the virtualenv. +``uv sync`` then builds numcodecs in editable mode (without build isolation, so +the venv's ``ninja`` is used for auto-rebuild on import). This two-step bootstrap +is needed because meson-python editable installs require build tools at runtime, +not just at build time. + +To run the tests:: + + $ uv run pytest -v -To verify that your development environment is working, you can run the unit tests:: +Setting up with venv and pip +""""""""""""""""""""""""""""" +If you prefer the standard library ``venv``:: + + $ python -m venv venv + $ source venv/bin/activate # macOS/Linux + $ # .\venv\Scripts\activate # Windows + + $ pip install meson-python meson ninja cython numpy setuptools-scm + $ pip install --no-build-isolation -e ".[test,test_extras,msgpack]" $ pytest -v +Passing build options +""""""""""""""""""""" + +NumCodecs uses `Meson `_ as its build system. You can +pass Meson options via pip's ``--config-settings``. + +To build against system-installed libraries instead of the vendored copies:: + + $ pip install --no-build-isolation -e . \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled + +To disable SIMD optimizations (e.g., for portable debugging):: + + $ pip install --no-build-isolation -e . \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled + +Rebuilding after changes +""""""""""""""""""""""""" + +With an editable install, meson-python automatically rebuilds changed C/Cython +extensions when you import ``numcodecs``. If you need a full clean rebuild:: + + $ rm -rf builddir + $ pip install --no-build-isolation -e . + Creating a branch ~~~~~~~~~~~~~~~~~ @@ -144,16 +205,17 @@ Running the test suite ~~~~~~~~~~~~~~~~~~~~~~ NumCodecs includes a suite of unit tests, as well as doctests included in function and class -docstrings. The simplest way to run the unit tests is to invoke:: +docstrings:: - $ pytest -v + $ uv run pytest -v + +Or, if using venv/pip:: -NumCodecs currently supports Python 6-3.9, so the above command must -succeed before code can be accepted into the main code base. + $ pytest -v -All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous -integration services for every pull request. Tests must pass under both services before -code can be accepted. +All tests are automatically run via GitHub Actions for every pull request across Linux +(x86_64, aarch64, i386), macOS (x86_64, arm64), and Windows (x86_64), with Python 3.11 +through 3.14. Tests must pass on all platforms before code can be accepted. Code standards ~~~~~~~~~~~~~~ @@ -167,11 +229,10 @@ Conformance can be checked by running:: Test coverage ~~~~~~~~~~~~~ -NumCodecs maintains 100% test coverage under the latest Python stable release (currently -Python 3.9). Both unit tests and docstring doctests are included when computing -coverage. Running ``pytest -v`` will automatically run the test suite with coverage -and produce a coverage report. This should be 100% before code can be accepted into the -main code base. +NumCodecs maintains 100% test coverage under the latest stable Python release. Both unit +tests and docstring doctests are included when computing coverage. Running ``pytest -v`` +will automatically run the test suite with coverage and produce a coverage report. This +should be 100% before code can be accepted into the main code base. When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull @@ -182,8 +243,7 @@ Documentation Docstrings for user-facing classes and functions should follow the `numpydoc `_ standard, -including sections for Parameters and Examples. All examples will be run as doctests -under Python 3.9. +including sections for Parameters and Examples. All examples will be run as doctests. NumCodecs uses Sphinx for documentation, hosted on readthedocs.org. Documentation is written in the RestructuredText markup language (.rst files) in the ``docs`` folder. diff --git a/docs/design/meson-python-migration.md b/docs/design/meson-python-migration.md new file mode 100644 index 000000000..1083e2445 --- /dev/null +++ b/docs/design/meson-python-migration.md @@ -0,0 +1,656 @@ +# Design Doc: Migrate numcodecs build system from setuptools/setup.py to meson-python + +**Status:** Proposal +**Date:** 2026-03-17 +**Author:** Max (with Claude) + +## Motivation + +numcodecs currently uses a hybrid build system: `pyproject.toml` declares the setuptools backend, but a 386-line `setup.py` contains all the C/Cython extension logic. This is problematic because: + +1. **`setup.py` is effectively deprecated.** PEP 517/518 moved the ecosystem toward declarative builds. `setup.py` is imperative, opaque to tooling, and executed during metadata extraction. +2. **`distutils` imports are fragile.** `setup.py` imports from `distutils` (removed from stdlib in Python 3.12). This only works because setuptools vendors a copy, which may be dropped in a future major release. +3. **CPU feature detection is incorrect for cross-compilation.** `py-cpuinfo` runs at build time on the *build* machine, not the *target* machine. This means cibuildwheel arm64 builds on x86_64 hosts detect x86_64 features. The current workaround is environment variables (`DISABLE_NUMCODECS_AVX2`, `DISABLE_NUMCODECS_SSE2`). +4. **System library linking is unsupported.** PR [#569](https://github.com/zarr-developers/numcodecs/pull/569) has been open since 2023, attempting to add `pkg-config`-based system library support. The approach adds complexity to an already complex `setup.py`. +5. **The scientific Python ecosystem has standardized on meson-python.** numpy, scipy, scikit-learn, scikit-image, and matplotlib have all migrated. Meson natively handles every concern currently implemented manually in `setup.py`. + +## Proposal + +Replace `setup.py` + setuptools with `meson.build` + `meson-python` as the build backend. Remove `py-cpuinfo` as a build dependency. Use meson's native compiler introspection for CPU feature detection and `dependency()` for optional system library linking. + +## Current Architecture + +### Build dependencies + +```toml +[build-system] +requires = ["setuptools>=77", "setuptools-scm[toml]>=6.2", "Cython", "py-cpuinfo", "numpy>2"] +build-backend = "setuptools.build_meta" +``` + +### Extension modules (8 total) + +| Extension | Cython source | C library deps | Assembly | Notes | +|-----------|--------------|----------------|----------|-------| +| `blosc` | `blosc.pyx` | c-blosc + vendored lz4, zlib, zstd | zstd `*amd64.S` / `*aarch64.S` | Most complex; SSE2/AVX2 conditional | +| `zstd` | `zstd.pyx` | vendored zstd (from c-blosc) | zstd `*amd64.S` / `*aarch64.S` | Shares zstd sources with blosc | +| `lz4` | `lz4.pyx` | vendored lz4 (from c-blosc) | None | | +| `vlen` | `vlen.pyx` | None | None | Needs numpy include dirs | +| `fletcher32` | `fletcher32.pyx` | None | None | | +| `jenkins` | `jenkins.pyx` | None | None | Has `CYTHON_TRACE=1` (likely accidental) | +| `compat_ext` | `compat_ext.pyx` | None | None | | +| `_shuffle` | `_shuffle.pyx` | None | None | | + +### Shared Cython declarations + +- `compat_ext.pxd` — `PyBytes_RESIZE`, `ensure_continguous_memoryview` (used by blosc, zstd, lz4, vlen) +- `_utils.pxd` — `store_le32`, `load_le32` (used by lz4, fletcher32, vlen) + +### Vendored C sources (c-blosc git submodule) + +``` +c-blosc/ +├── blosc/ # Core blosc library (28 files, incl. SIMD shuffle variants) +└── internal-complibs/ + ├── lz4-1.10.0/ # 4 files + ├── zlib-1.3.1/ # ~58 files + └── zstd-1.5.6/ # ~90 files across common/, compress/, decompress/, dictBuilder/ +``` + +### Conditional compilation logic in setup.py + +1. **CPU detection:** `py-cpuinfo` reads `/proc/cpuinfo` or equivalent → sets `have_sse2`, `have_avx2` +2. **Environment overrides:** `DISABLE_NUMCODECS_SSE2`, `DISABLE_NUMCODECS_AVX2`, `DISABLE_NUMCODECS_CEXT` +3. **SIMD sources:** SSE2/AVX2 shuffle/bitshuffle `.c` files included conditionally +4. **SIMD defines:** `-DSHUFFLE_SSE2_ENABLED`, `-DSHUFFLE_AVX2_ENABLED`, `__SSE2__`, `__AVX2__` +5. **Assembly:** `.S` files pre-compiled to `.o`, passed as `extra_objects` +6. **Platform flags:** `-stdlib=libc++` (macOS), `-pthread` (POSIX), `-Wno-implicit-function-declaration` (cibuildwheel macOS) + +## Proposed Architecture + +### Build dependencies + +```toml +[build-system] +requires = ["meson-python>=0.17", "meson>=1.6.0", "Cython>=3.0", "numpy>=2"] +build-backend = "mesonpy" +``` + +**Removed:** `setuptools`, `setuptools-scm`, `py-cpuinfo` + +### File structure + +``` +numcodecs/ +├── meson.build # Root: project(), subdir() +├── meson.options # Build options (system libs, SIMD toggles) +├── numcodecs/ +│ └── meson.build # Extension modules +├── c-blosc/ # Unchanged submodule +├── pyproject.toml # Updated build-backend +└── setup.py # DELETED +``` + +### Root meson.build + +```meson +project( + 'numcodecs', + 'c', 'cython', + version: run_command('python', '-m', 'setuptools_scm', '--version', check: true).stdout().strip(), + default_options: [ + 'c_std=c11', + 'warning_level=1', + ], + meson_version: '>=1.6.0', +) + +py = import('python').find_installation(pure: false) +cy = meson.get_compiler('cython') +cc = meson.get_compiler('c') + +# NumPy include directory (needed for vlen extension) +numpy_dep = dependency('numpy') + +subdir('numcodecs') +``` + +**Note on versioning:** meson-python has native setuptools-scm integration — see [meson-python docs on dynamic versioning](https://mesonbuild.com/meson-python/how-to-guides/dynamic-version.html). The `version` field can use `run_command` to call setuptools-scm, or we can switch to `meson-python`'s built-in VCS versioning. This will need to replace the current `write_to = "numcodecs/version.py"` approach — meson-python generates version metadata at build time via importlib.metadata instead of writing a file. + +### meson.options + +```meson +option('system_blosc', type: 'feature', value: 'auto', + description: 'Use system-installed Blosc library') +option('system_zstd', type: 'feature', value: 'auto', + description: 'Use system-installed Zstandard library') +option('system_lz4', type: 'feature', value: 'auto', + description: 'Use system-installed LZ4 library') +``` + +With `value: 'disabled'`, meson always uses the vendored sources by default, matching the old setup.py behavior. Users who want to link against system libraries opt in with `-Dsystem_blosc=enabled` etc. This replaces the `NUMCODECS_USE_SYSTEM_LIBS` env var from PR #569 with a standard meson idiom. + +### numcodecs/meson.build (core of the migration) + +This is the most complex file. The design below is broken into sections. + +#### Compiler flags + +```meson +# Platform-specific flags +c_args = [] +link_args = [] + +if host_machine.system() == 'darwin' + c_args += ['-stdlib=libc++'] +endif + +if host_machine.system() != 'windows' + c_args += ['-pthread'] + link_args += ['-pthread'] +endif +``` + +#### SIMD detection (replaces py-cpuinfo) + +```meson +# SIMD support — detected via compiler capability, NOT runtime CPU flags. +# This is correct for cross-compilation: we check what the *target* supports. +have_sse2 = false +have_avx2 = false + +if host_machine.cpu_family() == 'x86_64' + have_sse2 = cc.has_argument('-msse2') + have_avx2 = cc.has_argument('-mavx2') +endif + +# Allow disabling via meson configure (replaces DISABLE_NUMCODECS_* env vars) +# cibuildwheel can pass -Dsse2=disabled instead of DISABLE_NUMCODECS_SSE2=1 +``` + +This fixes the cross-compilation bug: `host_machine.cpu_family()` reflects the *target* architecture (from a meson cross file or native detection), not the build machine. + +#### Vendored libraries as static dependencies + +```meson +# --- Vendored zstd --- +zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + +if not zstd_dep.found() + zstd_sources = files( + # common/ + 'c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + 'c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + # ... (enumerate all .c files) + ) + + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files('c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S') + elif host_machine.cpu_family() == 'aarch64' + zstd_sources += files('c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_aarch64.S') + endif + + zstd_inc = include_directories( + 'c-blosc/internal-complibs/zstd-1.5.6', + 'c-blosc/internal-complibs/zstd-1.5.6/common', + 'c-blosc/internal-complibs/zstd-1.5.6/compress', + 'c-blosc/internal-complibs/zstd-1.5.6/decompress', + 'c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', + ) + + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, + c_args: c_args, + ) + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, + ) +endif + +# --- Vendored lz4 --- +lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + +if not lz4_dep.found() + lz4_sources = files( + 'c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + 'c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('c-blosc/internal-complibs/lz4-1.10.0') + + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, + ) +endif + +# --- Vendored zlib --- +zlib_sources = files( + 'c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + 'c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + 'c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + 'c-blosc/internal-complibs/zlib-1.3.1/trees.c', + 'c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + 'c-blosc/internal-complibs/zlib-1.3.1/zutil.c', +) +zlib_inc = include_directories('c-blosc/internal-complibs/zlib-1.3.1') + +zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, +) +zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, +) +``` + +#### Blosc extension (most complex) + +```meson +# --- Vendored blosc --- +blosc_dep = dependency('blosc', required: get_option('system_blosc')) + +if not blosc_dep.found() + blosc_sources = files( + 'c-blosc/blosc/blosc.c', + 'c-blosc/blosc/blosclz.c', + 'c-blosc/blosc/fastcopy.c', + 'c-blosc/blosc/shuffle.c', + 'c-blosc/blosc/shuffle-generic.c', + 'c-blosc/blosc/bitshuffle-generic.c', + ) + + blosc_c_args = c_args + blosc_c_args += ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + 'c-blosc/blosc/shuffle-sse2.c', + 'c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif + endif + + if have_avx2 + blosc_sources += files( + 'c-blosc/blosc/shuffle-avx2.c', + 'c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif + + blosc_inc = include_directories('c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) +endif +``` + +#### Extension module declarations + +```meson +# --- Cython extension modules --- + +# Extensions with C library dependencies +py.extension_module('blosc', + 'blosc.pyx', + dependencies: [blosc_dep, lz4_dep, zlib_dep, zstd_dep], + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('zstd', + 'zstd.pyx', + dependencies: [zstd_dep], + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('lz4', + 'lz4.pyx', + dependencies: [lz4_dep], + include_directories: include_directories('.'), + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('vlen', + 'vlen.pyx', + dependencies: [numpy_dep], + include_directories: include_directories('.'), + install: true, + subdir: 'numcodecs', +) + +# Pure-Cython extensions (no C library deps) +foreach ext : ['fletcher32', 'jenkins', 'compat_ext', '_shuffle'] + py.extension_module(ext, + ext + '.pyx', + include_directories: include_directories('.'), + install: true, + subdir: 'numcodecs', + ) +endforeach + +# --- Pure Python sources --- +py.install_sources( + '__init__.py', + 'abc.py', + 'compat.py', + 'registry.py', + # ... all .py files + subdir: 'numcodecs', +) +``` + +### Version management + +**Option A: meson-python + setuptools-scm (minimal change)** + +Keep `setuptools-scm` as a build dependency and use `run_command()` in `meson.build`: + +```meson +project('numcodecs', 'c', 'cython', + version: run_command( + py, ['-m', 'setuptools_scm', '--version'], check: true + ).stdout().strip(), +) +``` + +At runtime, version is read via `importlib.metadata.version('numcodecs')` instead of a generated `version.py`. Update `__init__.py`: + +```python +from importlib.metadata import version +__version__ = version("numcodecs") +``` + +**Option B: meson's built-in VCS tagging** + +Use `vcs_tag()` to generate a version file from git tags, removing the setuptools-scm dependency entirely. This is what numpy does. + +**Recommendation:** Option A — keep setuptools-scm for now, swap to `vcs_tag()` later if desired. + +### pyproject.toml changes + +```toml +[build-system] +requires = ["meson-python>=0.17", "meson>=1.6.0", "Cython>=3.0", "numpy>=2"] +build-backend = "mesonpy" + +[project] +# ... unchanged ... + +[tool.meson-python.args] +# Pass default meson options +setup = ['--default-library=static'] + +# Replace setuptools-scm config +[tool.setuptools_scm] # REMOVE this section + +# Replace setuptools config +[tool.setuptools] # REMOVE this section +``` + +### cibuildwheel changes + +The `DISABLE_NUMCODECS_*` environment variables are replaced by meson configure options: + +```toml +[tool.cibuildwheel] +# Meson cross-compilation handles SIMD correctly — no env vars needed +# AVX2 still disabled for portable wheels (not all x86_64 CPUs have it) +config-settings = "setup-args=-Davx2=disabled" + +[tool.cibuildwheel.macos] +# No need for MACOSX_DEPLOYMENT_TARGET — meson-python handles this +# No need for -Wno-implicit-function-declaration — meson compiles correctly + +[[tool.cibuildwheel.overrides]] +select = "*-macosx_arm64" +config-settings = "setup-args=-Davx2=disabled -Dsse2=disabled" +``` + +### Files deleted + +- `setup.py` (386 lines) +- `MANIFEST.in` (meson-python builds sdists from version control, like setuptools-scm) +- `numcodecs/version.py` (replaced by `importlib.metadata`) + +### Files added + +- `meson.build` (root, ~20 lines) +- `meson.options` (~15 lines) +- `numcodecs/meson.build` (~180 lines) + +### Files updated + +- `docs/contributing.rst` (rewritten — see [Contributing Guide Updates](#contributing-guide-updates) below) + +## Migration Plan + +numcodecs has limited maintenance bandwidth. A phased migration (the numpy/scipy approach of running two build systems in parallel for multiple releases) would double the maintenance surface during the transition. Instead, this should be a single PR that swaps everything at once. + +### Single-PR migration + +One PR, one release. The PR: + +1. **Adds** `meson.build`, `meson.options`, `numcodecs/meson.build` +2. **Deletes** `setup.py`, `MANIFEST.in` +3. **Updates** `pyproject.toml`: new `[build-system]`, removes `[tool.setuptools]`, `[tool.setuptools_scm]`, `[tool.setuptools.package-data]` +4. **Updates** `numcodecs/__init__.py`: version from `importlib.metadata` +5. **Updates** cibuildwheel config in `pyproject.toml` +6. **Updates** CI workflows (pixi tasks, wheel building) +7. **Updates** `.gitignore` (remove `numcodecs/version.py`) +8. **Updates** `docs/contributing.rst`: new dev setup instructions for pixi and uv, removes outdated references +9. **Closes** PR #569 (system library support is native via `-Dsystem_blosc=enabled`) + +### Validation checklist + +Before merging, verify: + +- [ ] `pip install .` works (non-editable) +- [ ] `pip install -e . --no-build-isolation` works (editable) +- [ ] `python -m build --sdist` produces correct sdist (includes c-blosc submodule) +- [ ] `python -m build --wheel` produces working wheel +- [ ] cibuildwheel produces wheels for all target platforms +- [ ] All tests pass on Linux x86_64, Linux aarch64, macOS arm64, macOS x86_64, Windows x86_64 +- [ ] i386 Alpine CI still works (may need a meson cross file) +- [ ] `pixi run run-tests` works for local development +- [ ] `-Dsystem_blosc=enabled` links against system blosc (replaces PR #569) +- [ ] Version string is correct in built package (`python -c "import numcodecs; print(numcodecs.__version__)"`) + +### Risk of a clean break + +The main risk is that a broken build blocks the next release. This is mitigated by: + +- **The test matrix is comprehensive.** CI already tests 5 OS/arch combos x 4 Python versions. If the meson build passes CI, it works. +- **meson-python is battle-tested.** numpy, scipy, scikit-learn all use it for projects with far more complex C/Cython builds than numcodecs. +- **Rollback is trivial.** If something goes wrong post-release, revert the PR and cut a patch release. The old `setup.py` is in git history. + +## Contributing Guide Updates + +The current `docs/contributing.rst` is significantly outdated (references Python 3.8/3.9, Travis CI, AppVeyor, bare `pip install -e .` without a venv manager). The meson migration is a natural point to rewrite the development setup section. + +### Current problems with the contributing guide + +- Recommends manual `python3 -m venv` + `pip install -e .` — works but requires the user to have a C compiler, Cython, and numpy already available, with no guidance on obtaining them +- No mention of pixi or uv, which are already configured in the project +- References Python 3.8, 3.9, Travis CI, and AppVeyor (all outdated) +- No mention of meson or how to pass build options +- No guidance on building against system libraries + +### Proposed "Creating a development environment" section + +The guide should present two supported paths (pixi and uv) and explicitly discourage developing without an isolated environment. The following replaces the "Creating a development environment" and "Running the test suite" sections. + +```rst +Creating a development environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NumCodecs contains C and Cython extensions, so you need a C compiler and +build tooling in addition to Python. We support two development workflows: +**pixi** (recommended) and **uv**. Both manage isolated environments and +are tested in CI. **Do not** install numcodecs into your system Python or a +manually-managed virtualenv — the C compilation dependencies are +difficult to get right by hand. + +Whichever tool you use, first clone the repository with its submodules:: + + $ git clone --recursive git@github.com:your-user-name/numcodecs.git + $ cd numcodecs + $ git remote add upstream https://github.com/zarr-developers/numcodecs.git + +Using pixi (recommended) +""""""""""""""""""""""""" + +`pixi `_ manages Python, the C compiler toolchain, and all +dependencies via conda-forge. This is the easiest way to get started, +especially on macOS or if you don't have a system C compiler. + +Install pixi, then:: + + $ pixi run -e test-py313 run-tests # build + test in one step + +This will create an isolated environment with Python 3.13, install the C +toolchain from conda-forge, build the Cython extensions, and run the test +suite. Available test environments:: + + test-py311, test-py312, test-py313, test-py314 + +To get a shell inside a pixi environment for interactive development:: + + $ pixi shell -e test-py313 + +Using uv +"""""""" + +`uv `_ is a fast Python package manager. It +manages virtualenvs and dependencies but does **not** provide a C compiler — +you must have ``cc`` / ``gcc`` / ``clang`` available on your ``PATH``. + +On Debian/Ubuntu:: + + $ sudo apt install build-essential + +On macOS (Xcode command line tools):: + + $ xcode-select --install + +Then:: + + $ uv sync --extra test --extra msgpack + $ uv run pytest -v + +uv will create a ``.venv``, build the extensions, and install numcodecs in +editable mode. + +Passing build options +""""""""""""""""""""" + +NumCodecs uses `meson `_ as its build system. You can +pass meson options via pip's ``--config-settings`` or via environment variables +when building. + +To build against system-installed Blosc, Zstd, and LZ4 instead of the +vendored copies:: + + $ pip install -e . --no-build-isolation \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled + +To disable SIMD optimizations (e.g. for portable debugging):: + + $ pip install -e . --no-build-isolation \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled + +Running the test suite +~~~~~~~~~~~~~~~~~~~~~~ + +With pixi:: + + $ pixi run -e test-py313 run-tests + +With uv:: + + $ uv run pytest -v + +To run tests for specific Zarr integration versions:: + + $ pixi run test-zarr-312 + $ pixi run test-zarr-313 + $ pixi run test-zarr-main + +All tests are automatically run via GitHub Actions for every pull request +across Linux (x86_64, aarch64, i386), macOS (x86_64, arm64), and Windows +(x86_64), with Python 3.11 through 3.14. +``` + +### Other sections to update + +The rest of the contributing guide needs lighter touch-ups as part of this PR: + +- **"Code standards"** — already correct (references `pre-commit run ruff`), no change needed +- **"Test coverage"** — update Python version reference from "3.9" to "3.13" (or just say "the latest stable release") +- **"Documentation"** — update Python version reference, otherwise fine +- **"Bug reports"** — update the Python session example from 3.8 to a current version +- **"Running the test suite" (old)** — replace entirely (covered above) +- Remove all references to Travis CI and AppVeyor — CI is GitHub Actions only + +## Risks and Mitigations + +### Risk: Meson doesn't support a platform/arch that setuptools did + +**Mitigation:** Meson supports all platforms numcodecs targets (Linux x86_64/aarch64/i386, macOS x86_64/arm64, Windows x86_64). The i386 Alpine CI job (`ci-i386.yml`) needs verification — meson may need a cross file for 32-bit builds inside the Alpine container. + +### Risk: Cython version incompatibilities + +**Mitigation:** Pin `Cython>=3.0` in build-requires. Meson's Cython support requires Cython 3. This drops Cython 0.x support, which is already effectively unsupported. + +### Risk: sdist contents change + +**Mitigation:** meson-python includes all files tracked by git (plus submodules) in sdists by default, matching setuptools-scm behavior. The `MANIFEST.in` is not needed. Verify with `python -m build --sdist` and compare contents. + +### Risk: Editable installs behave differently + +**Mitigation:** meson-python supports editable installs via `pip install -e . --no-build-isolation`. The experience differs from setuptools (meson-python uses import hooks rather than `.egg-link`). The pixi-based dev workflow should still work. Test editable installs explicitly in Phase 1. + +### Risk: Windows MSVC compatibility + +**Mitigation:** Meson has excellent MSVC support. The SIMD flag logic (`-msse2`, `-mavx2`) is GCC/Clang-specific; MSVC equivalents (`/arch:SSE2`, `/arch:AVX2`) are needed. Meson's `cc.get_id()` can branch on compiler. Current `setup.py` already handles this partially (the `__SSE2__`/`__AVX2__` defines for `os.name == 'nt'`). + +## References + +- [meson-python documentation](https://mesonbuild.com/meson-python/) +- [NumPy meson migration](https://numpy.org/doc/stable/reference/distutils_status_migration.html) +- [SciPy meson migration PR](https://github.com/scipy/scipy/pull/15959) +- [Meson Cython support](https://mesonbuild.com/Cython.html) +- [Meson SIMD module](https://mesonbuild.com/Simd-module.html) +- [PR #569 — system library linking](https://github.com/zarr-developers/numcodecs/pull/569) +- [Issue #464 — use system libraries](https://github.com/zarr-developers/numcodecs/issues/464) diff --git a/docs/index.rst b/docs/index.rst index cce02f0cd..06c46d1ec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,22 +36,25 @@ for AVX2.:: Note that if you compile the C extensions on a machine with AVX2 support you probably then cannot use the same binaries on a machine without AVX2. -If you specifically want to disable AVX2 or SSE2 when compiling, you can use -the following environment variables:: +If you specifically want to disable AVX2 or SSE2 when compiling from source, +you can pass meson build options:: - $ export DISABLE_NUMCODECS_AVX2=1 - $ export DISABLE_NUMCODECS_SSE2=1 + $ pip install -v --no-cache-dir --no-binary numcodecs numcodecs \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled +You can also build against system-installed Blosc, Zstd, and LZ4 libraries +instead of the vendored copies:: -To work with Numcodecs source code in development, clone the repository from GitHub -and then install in editable mode using `pip`.:: + $ pip install -v --no-cache-dir --no-binary numcodecs numcodecs \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled - $ git clone --recursive https://github.com/zarr-developers/numcodecs.git - $ cd numcodecs - $ pip install -e .[test,msgpack,zfpy] -Note: if you prefer to use the GitHub CLI ``gh`` you will need to append ``-- --recurse-submodules`` -to the clone command to everything works properly. +To work with Numcodecs source code in development, see the +`contributing guide `_ for instructions on setting up a +development environment with pixi or uv. To verify that Numcodecs has been fully installed (including the Blosc extension) run the test suite:: diff --git a/docs/release.rst b/docs/release.rst index fcd3422f1..c571bf590 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -17,6 +17,39 @@ Release notes Unreleased ---------- +Maintenance +~~~~~~~~~~~ + +* **Migrate build system from setuptools/setup.py to meson-python.** This replaces the + 386-line ``setup.py`` with declarative ``meson.build`` files. Benefits include correct + SIMD detection for cross-compilation, native support for linking against system-installed + Blosc/Zstd/LZ4 libraries (via ``-Dsystem_blosc=enabled`` etc.), and alignment with the + build system used by numpy, scipy, and scikit-learn. + + The ``DISABLE_NUMCODECS_AVX2`` and ``DISABLE_NUMCODECS_SSE2`` environment variables + continue to work for backwards compatibility. The preferred way to control SIMD is now + via meson options:: + + pip install numcodecs --no-binary numcodecs \ + --config-settings=setup-args=-Davx2=disabled + + The ``DISABLE_NUMCODECS_CEXT`` environment variable is no longer supported. + + By :user:`Max Jones `. + +* Move test suite from ``numcodecs/tests/`` to top-level ``tests/`` directory. This + avoids import shadowing issues with meson-python non-editable installs, where the + source tree's ``numcodecs/`` package (without compiled C extensions) would shadow the + installed package. This is the standard layout for meson-python projects (numpy, scipy, + scikit-learn all use top-level ``tests/``). Test imports change from + ``from numcodecs.tests.common import ...`` to ``from tests.common import ...``. + + By :user:`Max Jones `. + +* Rewrite contributing guide with pixi and uv development environment instructions. + + By :user:`Max Jones `. + .. _release_0.16.5: 0.16.5 diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..650d3a2cf --- /dev/null +++ b/meson.build @@ -0,0 +1,20 @@ +project( + 'numcodecs', + 'c', 'cython', + version: run_command( + 'python3', '-m', 'setuptools_scm', check: true, + ).stdout().strip(), + default_options: [ + 'c_std=gnu11', + 'warning_level=1', + ], + meson_version: '>=1.6.0', +) + +py = import('python').find_installation(pure: false) +cc = meson.get_compiler('c') + +# NumPy include directory (needed for vlen extension) +numpy_dep = dependency('numpy') + +subdir('src/numcodecs') diff --git a/meson.options b/meson.options new file mode 100644 index 000000000..26a2414d6 --- /dev/null +++ b/meson.options @@ -0,0 +1,10 @@ +option('system_blosc', type: 'feature', value: 'disabled', + description: 'Use system-installed Blosc library') +option('system_zstd', type: 'feature', value: 'disabled', + description: 'Use system-installed Zstandard library') +option('system_lz4', type: 'feature', value: 'disabled', + description: 'Use system-installed LZ4 library') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 SIMD optimizations') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 SIMD optimizations') diff --git a/numcodecs/tests/__init__.py b/numcodecs/tests/__init__.py deleted file mode 100644 index bb4e151d1..000000000 --- a/numcodecs/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import pytest - -pytest.register_assert_rewrite('numcodecs.tests.common') diff --git a/pyproject.toml b/pyproject.toml index a47600dc3..2d1445d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [build-system] requires = [ - "setuptools>=77", - "setuptools-scm[toml]>=6.2", - "Cython", - "py-cpuinfo", - "numpy>2", + "meson-python>=0.17", + "meson>=1.6.0", + "setuptools-scm>=6.2", + "Cython>=3.0", + "numpy>=2", ] -build-backend = "setuptools.build_meta" +build-backend = "mesonpy" [project] name = "numcodecs" @@ -80,6 +80,14 @@ test_extras = [ ] [dependency-groups] +dev = [ + "meson-python>=0.17", + "meson>=1.6.0", + "ninja", + "cython>=3.0", + "setuptools-scm>=6.2", + "numpy>=2", +] test-zarr-312 = [ "pytest", "pytest-cov", @@ -99,28 +107,12 @@ test-zarr-main = [ "crc32c", ] -[tool.setuptools] -package-dir = {"" = "."} -packages = ["numcodecs", "numcodecs.tests"] -zip-safe = false - -[tool.setuptools.package-data] -numcodecs = [ - "tests/package_with_entrypoint/__init__.py", - "tests/package_with_entrypoint-0.1.dist-info/entry_points.txt" -] - -[tool.setuptools_scm] -version_scheme = "guess-next-dev" -local_scheme = "dirty-tag" -write_to = "numcodecs/version.py" - [tool.codespell] skip = "./.git,fixture" ignore-words-list = "ba, compiletime, hist, nd, unparseable" [tool.coverage.run] -omit = ["numcodecs/tests/*"] +omit = ["tests/*"] [tool.coverage.report] exclude_lines = [ @@ -139,7 +131,7 @@ doctest_optionflags = [ "IGNORE_EXCEPTION_DETAIL", ] testpaths = [ - "numcodecs/tests", + "tests", ] norecursedirs = [ ".git", @@ -147,6 +139,8 @@ norecursedirs = [ ".pytest_cache", "adhoc", "build", + "builddir", + "subprojects", "c-blosc", "docs", "fixture", @@ -161,19 +155,16 @@ filterwarnings = [ ] [tool.cibuildwheel] -environment = { DISABLE_NUMCODECS_AVX2=1 } -[tool.cibuildwheel.macos] -# cibuildwheel uses 3.12 for the Python driver, which supports High Sierra and later -# https://github.com/pypa/cibuildwheel/blob/ee63bf16da6cddfb925f542f2c7b59ad50e93969/action.yml#L31 -environment = { MACOSX_DEPLOYMENT_TARGET=10.13, DISABLE_NUMCODECS_AVX2=1, CFLAGS="$CFLAGS -Wno-implicit-function-declaration" } +# AVX2 disabled for portable wheels (not all x86_64 CPUs have it) +config-settings = "setup-args=-Davx2=disabled" + [[tool.cibuildwheel.overrides]] select = "*-macosx_arm64" -environment = { DISABLE_NUMCODECS_AVX2=1, DISABLE_NUMCODECS_SSE2=1 } +config-settings = "setup-args=-Davx2=disabled -Dsse2=disabled" [tool.ruff] line-length = 100 extend-exclude = ["c-blosc"] -src = ["numcodecs"] [tool.ruff.lint] extend-select = [ @@ -230,7 +221,7 @@ ignore = [ ] [tool.ruff.lint.extend-per-file-ignores] -"numcodecs/tests/**" = ["SIM201", "SIM202", "SIM300", "TRY002"] +"tests/**" = ["SIM201", "SIM202", "SIM300", "TRY002"] "notebooks/**" = ["W391"] # https://github.com/astral-sh/ruff/issues/13763 [tool.ruff.format] @@ -248,6 +239,12 @@ warn_unused_ignores = true warn_unused_configs = true [tool.uv] +# meson-python editable installs require build deps (ninja, meson) at runtime for +# auto-rebuild on import. Skip build isolation so the venv's ninja is used. +# The "dev" dependency group provides these build tools. Bootstrap with: +# uv pip install meson-python meson ninja cython numpy setuptools-scm +# before running uv sync. +no-build-isolation-package = ["numcodecs"] conflicts = [ # Zarr versions conflict with each other [ @@ -287,9 +284,9 @@ test-google-crc32c = ["test", "test-google-crc32c"] ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze" ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze" ls-deps-main = "uv run --group test-zarr-main uv pip freeze" -test-zarr-312 = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" -test-zarr-313 = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" -test-zarr-main = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" +test-zarr-312 = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" +test-zarr-313 = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" +test-zarr-main = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" [tool.pixi.feature.test.tasks] run-tests = "pytest -v" diff --git a/setup.py b/setup.py deleted file mode 100644 index c27fad359..000000000 --- a/setup.py +++ /dev/null @@ -1,385 +0,0 @@ -import os -import sys -from distutils import ccompiler -from distutils.command.clean import clean -from distutils.sysconfig import customize_compiler -from glob import glob - -import cpuinfo -from Cython.Distutils.build_ext import new_build_ext as build_ext -from setuptools import Extension, setup -from setuptools.errors import CCompilerError, ExecError, PlatformError - -# determine CPU support for SSE2 and AVX2 -cpu_info = cpuinfo.get_cpu_info() -flags = cpu_info.get('flags', []) -machine = cpuinfo.platform.machine() - -# only check for x86 features on x86_64 arch -have_sse2 = False -have_avx2 = False -if machine == 'x86_64': - have_sse2 = 'sse2' in flags - have_avx2 = 'avx2' in flags - -disable_sse2 = 'DISABLE_NUMCODECS_SSE2' in os.environ -disable_avx2 = 'DISABLE_NUMCODECS_AVX2' in os.environ - -# setup common compile arguments -have_cflags = 'CFLAGS' in os.environ -base_compile_args = [] -if have_cflags: - # respect compiler options set by user - pass -elif os.name == 'posix' and machine == 'x86_64': - if disable_sse2: - base_compile_args.append('-mno-sse2') - elif have_sse2: - base_compile_args.append('-msse2') - if disable_avx2: - base_compile_args.append('-mno-avx2') - elif have_avx2: - base_compile_args.append('-mavx2') -# On macOS, force libc++ in case the system tries to use `stdlibc++`. -# The latter is often absent from modern macOS systems. -if sys.platform == 'darwin': - base_compile_args.append('-stdlib=libc++') - - -def info(*msg): - kwargs = {'file': sys.stdout} - print('[numcodecs]', *msg, **kwargs) - - -def error(*msg): - kwargs = {'file': sys.stderr} - print('[numcodecs]', *msg, **kwargs) - - -def blosc_extension(): - info('setting up Blosc extension') - - extra_compile_args = base_compile_args.copy() - extra_link_args = [] - define_macros = [] - - # ensure pthread is properly linked on POSIX systems - if os.name == 'posix': - extra_compile_args.append('-pthread') - extra_link_args.append('-pthread') - - # setup blosc sources - blosc_sources = [f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f] - include_dirs = [os.path.join('c-blosc', 'blosc')] - - # add internal complibs - blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c') - blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc') - blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*') if os.path.isdir(d)] - # remove minizip because Python.h 3.8 tries to include crypt.h - include_dirs = [d for d in include_dirs if 'minizip' not in d] - define_macros += [ - ('HAVE_LZ4', 1), - # ('HAVE_SNAPPY', 1), - ('HAVE_ZLIB', 1), - ('HAVE_ZSTD', 1), - ] - # define_macros += [('CYTHON_TRACE', '1')] - - # SSE2 - if have_sse2 and not disable_sse2: - info('compiling Blosc extension with SSE2 support') - extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED') - blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f] - if os.name == 'nt': - define_macros += [('__SSE2__', 1)] - else: - info('compiling Blosc extension without SSE2 support') - - # AVX2 - if have_avx2 and not disable_avx2: - info('compiling Blosc extension with AVX2 support') - extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') - blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] - if os.name == 'nt': - define_macros += [('__AVX2__', 1)] - else: - info('compiling Blosc extension without AVX2 support') - - # include assembly files - if cpuinfo.platform.machine() == 'x86_64': - extra_objects = [ - S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") - ] - else: - extra_objects = [] - - sources = ['numcodecs/blosc.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.blosc', - sources=sources + blosc_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - extra_objects=extra_objects, - ), - ] - - -def zstd_extension(): - info('setting up Zstandard extension') - - zstd_sources = [] - extra_compile_args = base_compile_args.copy() - include_dirs = [] - define_macros = [] - - # setup sources - use zstd bundled in blosc - zstd_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*/*') if os.path.isdir(d)] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/zstd.pyx'] - - # include assembly files - if cpuinfo.platform.machine() == 'x86_64': - extra_objects = [ - S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") - ] - else: - extra_objects = [] - - # define extension module - return [ - Extension( - 'numcodecs.zstd', - sources=sources + zstd_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_objects=extra_objects, - ), - ] - - -def lz4_extension(): - info('setting up LZ4 extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - use LZ4 bundled in blosc - lz4_sources = glob('c-blosc/internal-complibs/lz4*/*.c') - include_dirs = [d for d in glob('c-blosc/internal-complibs/lz4*') if os.path.isdir(d)] - include_dirs += ['numcodecs'] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/lz4.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.lz4', - sources=sources + lz4_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def vlen_extension(): - info('setting up vlen extension') - import numpy - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs', numpy.get_include()] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/vlen.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.vlen', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def fletcher_extension(): - info('setting up fletcher32 extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs'] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/fletcher32.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.fletcher32', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def jenkins_extension(): - info('setting up jenkins extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs'] - define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/jenkins.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.jenkins', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def compat_extension(): - info('setting up compat extension') - - extra_compile_args = base_compile_args.copy() - - sources = ['numcodecs/compat_ext.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.compat_ext', - sources=sources, - extra_compile_args=extra_compile_args, - ), - ] - - -def shuffle_extension(): - info('setting up shuffle extension') - - extra_compile_args = base_compile_args.copy() - - sources = ['numcodecs/_shuffle.pyx'] - - # define extension module - return [ - Extension('numcodecs._shuffle', sources=sources, extra_compile_args=extra_compile_args), - ] - - -if sys.platform == 'win32': - ext_errors = (CCompilerError, ExecError, PlatformError, IOError, ValueError) -else: - ext_errors = (CCompilerError, ExecError, PlatformError) - - -class BuildFailed(Exception): - pass - - -class ve_build_ext(build_ext): - # This class allows C extension building to fail. - - def run(self): - try: - machine = cpuinfo.platform.machine() - if machine in ('x86_64', 'aarch64'): - pattern = '*amd64.S' if machine == 'x86_64' else '*aarch64.S' - S_files = glob(f'c-blosc/internal-complibs/zstd*/decompress/{pattern}') - compiler = ccompiler.new_compiler() - customize_compiler(compiler) - compiler.src_extensions.append('.S') - compiler.compile(S_files) - - build_ext.run(self) - except PlatformError as e: - error(e) - raise BuildFailed from e - - def build_extension(self, ext): - try: - build_ext.build_extension(self, ext) - except ext_errors as e: - error(e) - raise BuildFailed from e - - -class Sclean(clean): - # Clean up .o files created by .S files - - def run(self): - if cpuinfo.platform.machine() == 'x86_64': - o_files = glob('c-blosc/internal-complibs/zstd*/decompress/*amd64.o') - for f in o_files: - os.remove(f) - - clean.run(self) - - -def run_setup(with_extensions): - if with_extensions: - ext_modules = ( - blosc_extension() - + zstd_extension() - + lz4_extension() - + compat_extension() - + shuffle_extension() - + vlen_extension() - + fletcher_extension() - + jenkins_extension() - ) - - cmdclass = {'build_ext': ve_build_ext, 'clean': Sclean} - else: - ext_modules = [] - cmdclass = {} - - setup( - ext_modules=ext_modules, - cmdclass=cmdclass, - ) - - -if __name__ == '__main__': - is_pypy = hasattr(sys, 'pypy_translation_info') - with_extensions = not is_pypy and 'DISABLE_NUMCODECS_CEXT' not in os.environ - run_setup(with_extensions) diff --git a/numcodecs/__init__.py b/src/numcodecs/__init__.py similarity index 97% rename from numcodecs/__init__.py rename to src/numcodecs/__init__.py index e3626b1b6..990341618 100644 --- a/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -20,10 +20,12 @@ import atexit import multiprocessing from contextlib import suppress +from importlib.metadata import version as _version from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec -from numcodecs.version import version as __version__ # noqa: F401 + +__version__: str = _version("numcodecs") from numcodecs.zlib import Zlib register_codec(Zlib) diff --git a/numcodecs/_shuffle.pyx b/src/numcodecs/_shuffle.pyx similarity index 100% rename from numcodecs/_shuffle.pyx rename to src/numcodecs/_shuffle.pyx diff --git a/numcodecs/_utils.pxd b/src/numcodecs/_utils.pxd similarity index 100% rename from numcodecs/_utils.pxd rename to src/numcodecs/_utils.pxd diff --git a/numcodecs/abc.py b/src/numcodecs/abc.py similarity index 100% rename from numcodecs/abc.py rename to src/numcodecs/abc.py diff --git a/numcodecs/astype.py b/src/numcodecs/astype.py similarity index 100% rename from numcodecs/astype.py rename to src/numcodecs/astype.py diff --git a/numcodecs/base64.py b/src/numcodecs/base64.py similarity index 100% rename from numcodecs/base64.py rename to src/numcodecs/base64.py diff --git a/numcodecs/bitround.py b/src/numcodecs/bitround.py similarity index 100% rename from numcodecs/bitround.py rename to src/numcodecs/bitround.py diff --git a/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx similarity index 100% rename from numcodecs/blosc.pyx rename to src/numcodecs/blosc.pyx diff --git a/numcodecs/bz2.py b/src/numcodecs/bz2.py similarity index 100% rename from numcodecs/bz2.py rename to src/numcodecs/bz2.py diff --git a/numcodecs/categorize.py b/src/numcodecs/categorize.py similarity index 100% rename from numcodecs/categorize.py rename to src/numcodecs/categorize.py diff --git a/numcodecs/checksum32.py b/src/numcodecs/checksum32.py similarity index 100% rename from numcodecs/checksum32.py rename to src/numcodecs/checksum32.py diff --git a/numcodecs/compat.py b/src/numcodecs/compat.py similarity index 100% rename from numcodecs/compat.py rename to src/numcodecs/compat.py diff --git a/numcodecs/compat_ext.pxd b/src/numcodecs/compat_ext.pxd similarity index 100% rename from numcodecs/compat_ext.pxd rename to src/numcodecs/compat_ext.pxd diff --git a/numcodecs/compat_ext.pyx b/src/numcodecs/compat_ext.pyx similarity index 100% rename from numcodecs/compat_ext.pyx rename to src/numcodecs/compat_ext.pyx diff --git a/numcodecs/delta.py b/src/numcodecs/delta.py similarity index 100% rename from numcodecs/delta.py rename to src/numcodecs/delta.py diff --git a/numcodecs/errors.py b/src/numcodecs/errors.py similarity index 100% rename from numcodecs/errors.py rename to src/numcodecs/errors.py diff --git a/numcodecs/fixedscaleoffset.py b/src/numcodecs/fixedscaleoffset.py similarity index 100% rename from numcodecs/fixedscaleoffset.py rename to src/numcodecs/fixedscaleoffset.py diff --git a/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx similarity index 100% rename from numcodecs/fletcher32.pyx rename to src/numcodecs/fletcher32.pyx diff --git a/numcodecs/gzip.py b/src/numcodecs/gzip.py similarity index 100% rename from numcodecs/gzip.py rename to src/numcodecs/gzip.py diff --git a/numcodecs/jenkins.pyx b/src/numcodecs/jenkins.pyx similarity index 100% rename from numcodecs/jenkins.pyx rename to src/numcodecs/jenkins.pyx diff --git a/numcodecs/json.py b/src/numcodecs/json.py similarity index 100% rename from numcodecs/json.py rename to src/numcodecs/json.py diff --git a/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx similarity index 100% rename from numcodecs/lz4.pyx rename to src/numcodecs/lz4.pyx diff --git a/numcodecs/lzma.py b/src/numcodecs/lzma.py similarity index 100% rename from numcodecs/lzma.py rename to src/numcodecs/lzma.py diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build new file mode 100644 index 000000000..2438a55af --- /dev/null +++ b/src/numcodecs/meson.build @@ -0,0 +1,290 @@ +# --- Compiler flags --- +c_args = [] +link_args = [] + +if host_machine.system() == 'darwin' + c_args += ['-stdlib=libc++'] +endif + +if host_machine.system() != 'windows' + c_args += ['-pthread'] + link_args += ['-pthread'] +endif + +# --- SIMD detection --- +# Detected via compiler capability, NOT runtime CPU flags. +# Correct for cross-compilation: checks what the *target* supports. +have_sse2 = false +have_avx2 = false + +if host_machine.cpu_family() == 'x86_64' + # Honor legacy environment variables for backwards compatibility with + # existing build scripts and packaging recipes. + disable_sse2_env = run_command( + 'python3', '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_SSE2", ""))', + check: true, + ).stdout().strip() != '' + disable_avx2_env = run_command( + 'python3', '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_AVX2", ""))', + check: true, + ).stdout().strip() != '' + + if get_option('sse2').disabled() or disable_sse2_env + have_sse2 = false + else + have_sse2 = cc.has_argument('-msse2') + endif + + if get_option('avx2').disabled() or disable_avx2_env + have_avx2 = false + else + have_avx2 = cc.has_argument('-mavx2') + endif +endif + +# --- Vendored zstd --- +zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + +if not zstd_dep.found() + zstd_sources = files( + # common/ + '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', + # compress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', + # decompress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', + # dictBuilder/ + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', + ) + + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files( + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', + ) + endif + + zstd_inc = include_directories( + '../../c-blosc/internal-complibs/zstd-1.5.6', + '../../c-blosc/internal-complibs/zstd-1.5.6/common', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', + ) + + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, + c_args: c_args, + ) + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, + ) +endif + +# --- Vendored lz4 --- +lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + +if not lz4_dep.found() + lz4_sources = files( + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') + + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, + ) +endif + +# --- Vendored zlib (always vendored, used by blosc) --- +zlib_sources = files( + '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', +) +zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') + +zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, +) +zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, +) + +# --- Vendored blosc --- +blosc_dep = dependency('blosc', required: get_option('system_blosc')) + +if not blosc_dep.found() + blosc_sources = files( + '../../c-blosc/blosc/blosc.c', + '../../c-blosc/blosc/blosclz.c', + '../../c-blosc/blosc/fastcopy.c', + '../../c-blosc/blosc/shuffle.c', + '../../c-blosc/blosc/shuffle-generic.c', + '../../c-blosc/blosc/bitshuffle-generic.c', + ) + + blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-sse2.c', + '../../c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif + endif + + if have_avx2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-avx2.c', + '../../c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif + + blosc_inc = include_directories('../../c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) +endif + +# --- Cython extension modules --- + +# Extensions with C library dependencies +py.extension_module('blosc', + 'blosc.pyx', + dependencies: [blosc_dep, lz4_dep, zlib_dep, zstd_dep], + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('zstd', + 'zstd.pyx', + dependencies: [zstd_dep], + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('lz4', + 'lz4.pyx', + dependencies: [lz4_dep], + include_directories: include_directories('.'), + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('vlen', + 'vlen.pyx', + dependencies: [numpy_dep], + include_directories: include_directories('.'), + c_args: c_args, + install: true, + subdir: 'numcodecs', +) + +# Pure-Cython extensions (no C library deps) +foreach ext : ['fletcher32', 'jenkins', 'compat_ext', '_shuffle'] + py.extension_module(ext, + ext + '.pyx', + include_directories: include_directories('.'), + c_args: c_args, + install: true, + subdir: 'numcodecs', + ) +endforeach + +# --- Pure Python sources --- +py.install_sources( + '__init__.py', + 'abc.py', + 'astype.py', + 'base64.py', + 'bitround.py', + 'bz2.py', + 'categorize.py', + 'checksum32.py', + 'compat.py', + 'delta.py', + 'errors.py', + 'fixedscaleoffset.py', + 'gzip.py', + 'json.py', + 'lzma.py', + 'msgpacks.py', + 'ndarray_like.py', + 'packbits.py', + 'pcodec.py', + 'pickles.py', + 'quantize.py', + 'registry.py', + 'shuffle.py', + 'version.py', + 'zarr3.py', + 'zfpy.py', + 'zlib.py', + subdir: 'numcodecs', +) diff --git a/numcodecs/msgpacks.py b/src/numcodecs/msgpacks.py similarity index 100% rename from numcodecs/msgpacks.py rename to src/numcodecs/msgpacks.py diff --git a/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py similarity index 100% rename from numcodecs/ndarray_like.py rename to src/numcodecs/ndarray_like.py diff --git a/numcodecs/packbits.py b/src/numcodecs/packbits.py similarity index 100% rename from numcodecs/packbits.py rename to src/numcodecs/packbits.py diff --git a/numcodecs/pcodec.py b/src/numcodecs/pcodec.py similarity index 99% rename from numcodecs/pcodec.py rename to src/numcodecs/pcodec.py index e214f7812..79732f2fd 100644 --- a/numcodecs/pcodec.py +++ b/src/numcodecs/pcodec.py @@ -1,8 +1,9 @@ from typing import Literal +from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone + from numcodecs.abc import Codec from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray -from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone DEFAULT_MAX_PAGE_N = 262144 diff --git a/numcodecs/pickles.py b/src/numcodecs/pickles.py similarity index 100% rename from numcodecs/pickles.py rename to src/numcodecs/pickles.py diff --git a/numcodecs/quantize.py b/src/numcodecs/quantize.py similarity index 100% rename from numcodecs/quantize.py rename to src/numcodecs/quantize.py diff --git a/numcodecs/registry.py b/src/numcodecs/registry.py similarity index 100% rename from numcodecs/registry.py rename to src/numcodecs/registry.py diff --git a/numcodecs/shuffle.py b/src/numcodecs/shuffle.py similarity index 100% rename from numcodecs/shuffle.py rename to src/numcodecs/shuffle.py diff --git a/src/numcodecs/version.py b/src/numcodecs/version.py new file mode 100644 index 000000000..b69bb5a28 --- /dev/null +++ b/src/numcodecs/version.py @@ -0,0 +1,7 @@ +# Backwards-compatibility shim. The version is now provided by importlib.metadata +# rather than being generated by setuptools-scm. This module exists so that +# `from numcodecs.version import version` continues to work. +from importlib.metadata import version as _version + +__version__: str = _version("numcodecs") +version: str = __version__ diff --git a/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx similarity index 100% rename from numcodecs/vlen.pyx rename to src/numcodecs/vlen.pyx diff --git a/numcodecs/zarr3.py b/src/numcodecs/zarr3.py similarity index 100% rename from numcodecs/zarr3.py rename to src/numcodecs/zarr3.py diff --git a/numcodecs/zfpy.py b/src/numcodecs/zfpy.py similarity index 100% rename from numcodecs/zfpy.py rename to src/numcodecs/zfpy.py diff --git a/numcodecs/zlib.py b/src/numcodecs/zlib.py similarity index 100% rename from numcodecs/zlib.py rename to src/numcodecs/zlib.py diff --git a/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx similarity index 100% rename from numcodecs/zstd.pyx rename to src/numcodecs/zstd.pyx diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..76094f9ee --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite('tests.common') diff --git a/numcodecs/tests/common.py b/tests/common.py similarity index 100% rename from numcodecs/tests/common.py rename to tests/common.py diff --git a/numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt similarity index 100% rename from numcodecs/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt rename to tests/package_with_entrypoint-0.1.dist-info/entry_points.txt diff --git a/numcodecs/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py similarity index 100% rename from numcodecs/tests/package_with_entrypoint/__init__.py rename to tests/package_with_entrypoint/__init__.py diff --git a/numcodecs/tests/test_astype.py b/tests/test_astype.py similarity index 98% rename from numcodecs/tests/test_astype.py rename to tests/test_astype.py index b4653f5be..4022e2f5e 100644 --- a/numcodecs/tests/test_astype.py +++ b/tests/test_astype.py @@ -2,7 +2,7 @@ from numpy.testing import assert_array_equal from numcodecs.astype import AsType -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_base64.py b/tests/test_base64.py similarity index 98% rename from numcodecs/tests/test_base64.py rename to tests/test_base64.py index 03f2d491c..39d118034 100644 --- a/numcodecs/tests/test_base64.py +++ b/tests/test_base64.py @@ -4,7 +4,7 @@ import pytest from numcodecs.base64 import Base64 -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_encode_decode, check_err_decode_object_buffer, diff --git a/numcodecs/tests/test_bitround.py b/tests/test_bitround.py similarity index 100% rename from numcodecs/tests/test_bitround.py rename to tests/test_bitround.py diff --git a/numcodecs/tests/test_blosc.py b/tests/test_blosc.py similarity index 99% rename from numcodecs/tests/test_blosc.py rename to tests/test_blosc.py index 52ef183a4..537dc3505 100644 --- a/numcodecs/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -5,13 +5,14 @@ import pytest try: - from numcodecs import blosc from numcodecs.blosc import Blosc + + from numcodecs import blosc except ImportError: # pragma: no cover pytest.skip("numcodecs.blosc not available", allow_module_level=True) -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_bz2.py b/tests/test_bz2.py similarity index 98% rename from numcodecs/tests/test_bz2.py rename to tests/test_bz2.py index ce83f1e2d..74ed5534e 100644 --- a/numcodecs/tests/test_bz2.py +++ b/tests/test_bz2.py @@ -3,7 +3,7 @@ import numpy as np from numcodecs.bz2 import BZ2 -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_categorize.py b/tests/test_categorize.py similarity index 98% rename from numcodecs/tests/test_categorize.py rename to tests/test_categorize.py index 0ec115bb3..97aca9770 100644 --- a/numcodecs/tests/test_categorize.py +++ b/tests/test_categorize.py @@ -3,7 +3,7 @@ from numpy.testing import assert_array_equal from numcodecs.categorize import Categorize -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_checksum32.py b/tests/test_checksum32.py similarity index 99% rename from numcodecs/tests/test_checksum32.py rename to tests/test_checksum32.py index 06b0a1a8c..0a6db0c7d 100644 --- a/numcodecs/tests/test_checksum32.py +++ b/tests/test_checksum32.py @@ -6,7 +6,7 @@ import pytest from numcodecs.checksum32 import CRC32, Adler32 -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_compat.py b/tests/test_compat.py similarity index 100% rename from numcodecs/tests/test_compat.py rename to tests/test_compat.py diff --git a/numcodecs/tests/test_delta.py b/tests/test_delta.py similarity index 97% rename from numcodecs/tests/test_delta.py rename to tests/test_delta.py index 9664efeee..df48d6d06 100644 --- a/numcodecs/tests/test_delta.py +++ b/tests/test_delta.py @@ -3,7 +3,7 @@ from numpy.testing import assert_array_equal from numcodecs.delta import Delta -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_entrypoints.py b/tests/test_entrypoints.py similarity index 100% rename from numcodecs/tests/test_entrypoints.py rename to tests/test_entrypoints.py diff --git a/numcodecs/tests/test_entrypoints_backport.py b/tests/test_entrypoints_backport.py similarity index 100% rename from numcodecs/tests/test_entrypoints_backport.py rename to tests/test_entrypoints_backport.py diff --git a/numcodecs/tests/test_fixedscaleoffset.py b/tests/test_fixedscaleoffset.py similarity index 98% rename from numcodecs/tests/test_fixedscaleoffset.py rename to tests/test_fixedscaleoffset.py index 337e893ee..343d30263 100644 --- a/numcodecs/tests/test_fixedscaleoffset.py +++ b/tests/test_fixedscaleoffset.py @@ -5,7 +5,7 @@ from numpy.testing import assert_array_equal from numcodecs.fixedscaleoffset import FixedScaleOffset -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_fletcher32.py b/tests/test_fletcher32.py similarity index 99% rename from numcodecs/tests/test_fletcher32.py rename to tests/test_fletcher32.py index 3bbefe850..adfb31da6 100644 --- a/numcodecs/tests/test_fletcher32.py +++ b/tests/test_fletcher32.py @@ -1,6 +1,5 @@ import numpy as np import pytest - from numcodecs.fletcher32 import Fletcher32 diff --git a/numcodecs/tests/test_gzip.py b/tests/test_gzip.py similarity index 98% rename from numcodecs/tests/test_gzip.py rename to tests/test_gzip.py index d63676412..5031231e7 100644 --- a/numcodecs/tests/test_gzip.py +++ b/tests/test_gzip.py @@ -4,7 +4,7 @@ import pytest from numcodecs.gzip import GZip -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_jenkins.py b/tests/test_jenkins.py similarity index 100% rename from numcodecs/tests/test_jenkins.py rename to tests/test_jenkins.py index de008be52..40d972bf7 100644 --- a/numcodecs/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -1,8 +1,8 @@ import numpy as np import pytest +from numcodecs.jenkins import jenkins_lookup3 from numcodecs.checksum32 import JenkinsLookup3 -from numcodecs.jenkins import jenkins_lookup3 def test_jenkins_lookup3(): diff --git a/numcodecs/tests/test_json.py b/tests/test_json.py similarity index 98% rename from numcodecs/tests/test_json.py rename to tests/test_json.py index 771a05191..c8b2cbca0 100644 --- a/numcodecs/tests/test_json.py +++ b/tests/test_json.py @@ -4,7 +4,7 @@ import pytest from numcodecs.json import JSON -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode_array, diff --git a/numcodecs/tests/test_lz4.py b/tests/test_lz4.py similarity index 98% rename from numcodecs/tests/test_lz4.py rename to tests/test_lz4.py index 82af464af..c6b8faec9 100644 --- a/numcodecs/tests/test_lz4.py +++ b/tests/test_lz4.py @@ -9,7 +9,7 @@ pytest.skip("numcodecs.lz4 not available", allow_module_level=True) -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_lzma.py b/tests/test_lzma.py similarity index 98% rename from numcodecs/tests/test_lzma.py rename to tests/test_lzma.py index 017f6da65..ff36cc428 100644 --- a/numcodecs/tests/test_lzma.py +++ b/tests/test_lzma.py @@ -13,7 +13,7 @@ raise unittest.SkipTest("LZMA not available") from e -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_msgpacks.py b/tests/test_msgpacks.py similarity index 99% rename from numcodecs/tests/test_msgpacks.py rename to tests/test_msgpacks.py index 4f8778459..0792cdfdd 100644 --- a/numcodecs/tests/test_msgpacks.py +++ b/tests/test_msgpacks.py @@ -9,7 +9,7 @@ raise unittest.SkipTest("msgpack not available") from e -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode_array, diff --git a/numcodecs/tests/test_ndarray_like.py b/tests/test_ndarray_like.py similarity index 100% rename from numcodecs/tests/test_ndarray_like.py rename to tests/test_ndarray_like.py diff --git a/numcodecs/tests/test_packbits.py b/tests/test_packbits.py similarity index 96% rename from numcodecs/tests/test_packbits.py rename to tests/test_packbits.py index 303216172..8bda1cd33 100644 --- a/numcodecs/tests/test_packbits.py +++ b/tests/test_packbits.py @@ -1,7 +1,7 @@ import numpy as np from numcodecs.packbits import PackBits -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_pcodec.py b/tests/test_pcodec.py similarity index 98% rename from numcodecs/tests/test_pcodec.py rename to tests/test_pcodec.py index d0520b45e..e34d66589 100644 --- a/numcodecs/tests/test_pcodec.py +++ b/tests/test_pcodec.py @@ -6,7 +6,7 @@ except ImportError: # pragma: no cover pytest.skip("pcodec not available", allow_module_level=True) -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode_array, diff --git a/numcodecs/tests/test_pickles.py b/tests/test_pickles.py similarity index 97% rename from numcodecs/tests/test_pickles.py rename to tests/test_pickles.py index 6f7ad2f7b..fdabd459c 100644 --- a/numcodecs/tests/test_pickles.py +++ b/tests/test_pickles.py @@ -5,7 +5,7 @@ import pytest from numcodecs.pickles import Pickle -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode_array, diff --git a/numcodecs/tests/test_pyzstd.py b/tests/test_pyzstd.py similarity index 99% rename from numcodecs/tests/test_pyzstd.py rename to tests/test_pyzstd.py index 7ee6084b4..e6df84a71 100644 --- a/numcodecs/tests/test_pyzstd.py +++ b/tests/test_pyzstd.py @@ -3,7 +3,6 @@ import numpy as np import pytest import pyzstd - from numcodecs.zstd import Zstd test_data = [ diff --git a/numcodecs/tests/test_quantize.py b/tests/test_quantize.py similarity index 98% rename from numcodecs/tests/test_quantize.py rename to tests/test_quantize.py index c070ed72b..724a7ba82 100644 --- a/numcodecs/tests/test_quantize.py +++ b/tests/test_quantize.py @@ -5,7 +5,7 @@ from numpy.testing import assert_array_almost_equal, assert_array_equal from numcodecs.quantize import Quantize -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_registry.py b/tests/test_registry.py similarity index 100% rename from numcodecs/tests/test_registry.py rename to tests/test_registry.py diff --git a/numcodecs/tests/test_shuffle.py b/tests/test_shuffle.py similarity index 99% rename from numcodecs/tests/test_shuffle.py rename to tests/test_shuffle.py index 6e6d744a1..8035c5586 100644 --- a/numcodecs/tests/test_shuffle.py +++ b/tests/test_shuffle.py @@ -10,7 +10,7 @@ pytest.skip("numcodecs.shuffle not available", allow_module_level=True) -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, diff --git a/numcodecs/tests/test_vlen_array.py b/tests/test_vlen_array.py similarity index 98% rename from numcodecs/tests/test_vlen_array.py rename to tests/test_vlen_array.py index 79d6be42d..f25374a06 100644 --- a/numcodecs/tests/test_vlen_array.py +++ b/tests/test_vlen_array.py @@ -7,7 +7,7 @@ from numcodecs.vlen import VLenArray except ImportError as e: # pragma: no cover raise unittest.SkipTest("vlen-array not available") from e -from numcodecs.tests.common import ( +from tests.common import ( assert_array_items_equal, check_backwards_compatibility, check_config, diff --git a/numcodecs/tests/test_vlen_bytes.py b/tests/test_vlen_bytes.py similarity index 98% rename from numcodecs/tests/test_vlen_bytes.py rename to tests/test_vlen_bytes.py index 3546dbaa7..facd09472 100644 --- a/numcodecs/tests/test_vlen_bytes.py +++ b/tests/test_vlen_bytes.py @@ -7,7 +7,7 @@ from numcodecs.vlen import VLenBytes except ImportError as e: # pragma: no cover raise unittest.SkipTest("vlen-bytes not available") from e -from numcodecs.tests.common import ( +from tests.common import ( assert_array_items_equal, check_backwards_compatibility, check_config, diff --git a/numcodecs/tests/test_vlen_utf8.py b/tests/test_vlen_utf8.py similarity index 98% rename from numcodecs/tests/test_vlen_utf8.py rename to tests/test_vlen_utf8.py index 514c7dc16..ee54793cb 100644 --- a/numcodecs/tests/test_vlen_utf8.py +++ b/tests/test_vlen_utf8.py @@ -7,7 +7,7 @@ from numcodecs.vlen import VLenUTF8 except ImportError as e: # pragma: no cover raise unittest.SkipTest("vlen-utf8 not available") from e -from numcodecs.tests.common import ( +from tests.common import ( assert_array_items_equal, check_backwards_compatibility, check_config, diff --git a/numcodecs/tests/test_zarr3.py b/tests/test_zarr3.py similarity index 100% rename from numcodecs/tests/test_zarr3.py rename to tests/test_zarr3.py diff --git a/numcodecs/tests/test_zarr3_import.py b/tests/test_zarr3_import.py similarity index 100% rename from numcodecs/tests/test_zarr3_import.py rename to tests/test_zarr3_import.py diff --git a/numcodecs/tests/test_zfpy.py b/tests/test_zfpy.py similarity index 98% rename from numcodecs/tests/test_zfpy.py rename to tests/test_zfpy.py index e48c3cb7d..3c15aacb8 100644 --- a/numcodecs/tests/test_zfpy.py +++ b/tests/test_zfpy.py @@ -11,7 +11,7 @@ pytest.skip("ZFPY not available", allow_module_level=True) -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode_array, diff --git a/numcodecs/tests/test_zlib.py b/tests/test_zlib.py similarity index 98% rename from numcodecs/tests/test_zlib.py rename to tests/test_zlib.py index 15f1ef587..82e692b7f 100644 --- a/numcodecs/tests/test_zlib.py +++ b/tests/test_zlib.py @@ -3,7 +3,8 @@ import numpy as np import pytest -from numcodecs.tests.common import ( +from numcodecs.zlib import Zlib +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, @@ -11,7 +12,6 @@ check_err_encode_object_buffer, check_repr, ) -from numcodecs.zlib import Zlib codecs = [ Zlib(), diff --git a/numcodecs/tests/test_zstd.py b/tests/test_zstd.py similarity index 99% rename from numcodecs/tests/test_zstd.py rename to tests/test_zstd.py index a3a926ebd..09342088a 100644 --- a/numcodecs/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -3,8 +3,9 @@ import numpy as np import pytest +from numcodecs.zstd import Zstd -from numcodecs.tests.common import ( +from tests.common import ( check_backwards_compatibility, check_config, check_encode_decode, @@ -12,7 +13,6 @@ check_err_encode_object_buffer, check_repr, ) -from numcodecs.zstd import Zstd codecs = [ Zstd(), From 27989e1197a0d680b3cd7d9034b5cffc93504e6d Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 21:59:27 -0400 Subject: [PATCH 02/23] Bootstrap first --- .github/workflows/ci-i386.yml | 3 ++- adhoc/blosc_memleak_check.py | 3 ++- notebooks/benchmark_vlen.ipynb | 3 ++- pyproject.toml | 13 +++++++------ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 317a5c4d2..194fabdbd 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -62,7 +62,8 @@ jobs: - name: Install numcodecs run: | uv venv - uv pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + uv pip install meson-python meson ninja cython numpy setuptools-scm + uv pip install --no-build-isolation -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" shell: alpine.sh {0} - name: List installed packages diff --git a/adhoc/blosc_memleak_check.py b/adhoc/blosc_memleak_check.py index a6409ed19..427c3e153 100644 --- a/adhoc/blosc_memleak_check.py +++ b/adhoc/blosc_memleak_check.py @@ -1,9 +1,10 @@ import sys -import numcodecs import numpy as np from numpy.testing import assert_array_equal +import numcodecs + codec = numcodecs.Blosc() data = np.arange(int(sys.argv[1])) for _ in range(int(sys.argv[2])): diff --git a/notebooks/benchmark_vlen.ipynb b/notebooks/benchmark_vlen.ipynb index cb971278e..c627cac95 100644 --- a/notebooks/benchmark_vlen.ipynb +++ b/notebooks/benchmark_vlen.ipynb @@ -24,9 +24,10 @@ } ], "source": [ - "import numcodecs\n", "import numpy as np\n", "\n", + "import numcodecs\n", + "\n", "numcodecs.__version__" ] }, diff --git a/pyproject.toml b/pyproject.toml index 2d1445d23..eb2329dfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -281,12 +281,13 @@ test-crc32c = ["test", "test-crc32c"] test-google-crc32c = ["test", "test-google-crc32c"] [tool.pixi.tasks] -ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze" -ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze" -ls-deps-main = "uv run --group test-zarr-main uv pip freeze" -test-zarr-312 = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" -test-zarr-313 = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" -test-zarr-main = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" +bootstrap = "uv pip install meson-python meson ninja cython numpy setuptools-scm" +ls-deps-312 = { cmd = "uv run --group test-zarr-312 uv pip freeze", depends-on = ["bootstrap"] } +ls-deps-313 = { cmd = "uv run --group test-zarr-313 uv pip freeze", depends-on = ["bootstrap"] } +ls-deps-main = { cmd = "uv run --group test-zarr-main uv pip freeze", depends-on = ["bootstrap"] } +test-zarr-312 = { cmd = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } +test-zarr-313 = { cmd = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } +test-zarr-main = { cmd = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } [tool.pixi.feature.test.tasks] run-tests = "pytest -v" From de0a7a1c7dadcb9ffe7192369eb0d8992d776887 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:03:24 -0400 Subject: [PATCH 03/23] Combine installs --- .github/workflows/ci-i386.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 194fabdbd..0bd028ccf 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -45,9 +45,10 @@ jobs: py3-pip py3-pytest - - name: Install zfp + - name: Install zfp and numcodecs run: | uv venv + uv pip install meson-python meson ninja cython numpy setuptools-scm PYTHON_INCLUDE=$(uv run python -c 'from sysconfig import get_paths; print(get_paths()["include"])'); PYTHON_LIB=$(uv run python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))'); git clone https://github.com/LLNL/zfp @@ -56,13 +57,7 @@ jobs: uv run cmake zfp -B zfp/build -DBUILD_ZFPY=ON -DBUILD_TESTING=OFF -DPYTHON_LIBRARY=$PYTHON_LIB -DPYTHON_INCLUDE_DIR=$PYTHON_INCLUDE uv run make -j -C zfp/build uv run sudo make -C zfp/build install - uv add "zfpy @ ./zfp" - shell: alpine.sh {0} - - - name: Install numcodecs - run: | - uv venv - uv pip install meson-python meson ninja cython numpy setuptools-scm + uv pip install ./zfp uv pip install --no-build-isolation -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" shell: alpine.sh {0} From f46297e108617832b7654f4add7c37642b8d74c6 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:03:31 -0400 Subject: [PATCH 04/23] Drop pixi --- .github/workflows/ci.yaml | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c65c45aa9..863a72081 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,7 +74,13 @@ jobs: strategy: fail-fast: false matrix: - zarr-version: ["312", "313", "main"] + include: + - zarr-version: "312" + zarr-pkg: "zarr==3.1.2" + - zarr-version: "313" + zarr-pkg: "zarr==3.1.3" + - zarr-version: "main" + zarr-pkg: "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main" defaults: run: @@ -87,19 +93,27 @@ jobs: submodules: recursive fetch-depth: 0 # required for version resolution - - name: Set up Pixi - uses: prefix-dev/setup-pixi@v0.9.0 + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3.2.0 with: - pixi-version: v0.49.0 - cache: false + channels: conda-forge + miniforge-version: latest + python-version: "3.13" + + - name: Install compilers + run: conda install -y c-compiler cxx-compiler + + - name: Install numcodecs and Zarr + run: | + python -m pip install meson-python meson ninja cython numpy setuptools-scm + python -m pip install --no-build-isolation -v ".[test,test_extras]" + python -m pip install "${{ matrix.zarr-pkg }}" crc32c - name: List installed packages - shell: "bash -l {0}" - run: pixi run ls-deps-${{matrix.zarr-version}} + run: python -m pip list - - name: Run tests with Zarr ${{ matrix.zarr-version }} - shell: "bash -l {0}" - run: pixi run test-zarr-${{ matrix.zarr-version }} + - name: Run Zarr integration tests + run: pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py - uses: codecov/codecov-action@v5 with: From f7c85f31738e9c8b55919ade11b88c5fea683531 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:05:46 -0400 Subject: [PATCH 05/23] Seperate args --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eb2329dfa..9f818b30c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,7 +160,7 @@ config-settings = "setup-args=-Davx2=disabled" [[tool.cibuildwheel.overrides]] select = "*-macosx_arm64" -config-settings = "setup-args=-Davx2=disabled -Dsse2=disabled" +config-settings = {setup-args = ["-Davx2=disabled", "-Dsse2=disabled"]} [tool.ruff] line-length = 100 From 85549c476aadeb043607c6221ac91a2b964a8efc Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:17:50 -0400 Subject: [PATCH 06/23] fix for windows --- meson.build | 2 +- src/numcodecs/_build_utils/gitversion.py | 26 ++++++++++++++++++++++++ src/numcodecs/meson.build | 4 ++-- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100755 src/numcodecs/_build_utils/gitversion.py diff --git a/meson.build b/meson.build index 650d3a2cf..8d90d9c78 100644 --- a/meson.build +++ b/meson.build @@ -2,7 +2,7 @@ project( 'numcodecs', 'c', 'cython', version: run_command( - 'python3', '-m', 'setuptools_scm', check: true, + ['src/numcodecs/_build_utils/gitversion.py'], check: true, ).stdout().strip(), default_options: [ 'c_std=gnu11', diff --git a/src/numcodecs/_build_utils/gitversion.py b/src/numcodecs/_build_utils/gitversion.py new file mode 100755 index 000000000..32c5165dc --- /dev/null +++ b/src/numcodecs/_build_utils/gitversion.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Get the version string for numcodecs, used by meson.build at configure time.""" + +import os + + +def get_version(): + try: + from setuptools_scm import get_version + + return get_version(root=os.path.join(os.path.dirname(__file__), '..', '..', '..')) + except (ImportError, LookupError): + pass + + # Fallback: read from pyproject.toml + pyproject = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'pyproject.toml') + with open(pyproject) as f: + for line in f: + if line.strip().startswith('version'): + return line.split('=')[1].strip().strip('"').strip("'") + + return '0.0.0' + + +if __name__ == '__main__': + print(get_version()) diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 2438a55af..ab67a49b6 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -21,11 +21,11 @@ if host_machine.cpu_family() == 'x86_64' # Honor legacy environment variables for backwards compatibility with # existing build scripts and packaging recipes. disable_sse2_env = run_command( - 'python3', '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_SSE2", ""))', + py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_SSE2", ""))', check: true, ).stdout().strip() != '' disable_avx2_env = run_command( - 'python3', '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_AVX2", ""))', + py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_AVX2", ""))', check: true, ).stdout().strip() != '' From 0542e15d489aae3fd35a1a62f048cc4f35f62bb7 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:18:30 -0400 Subject: [PATCH 07/23] Fix bootstrap --- .github/workflows/ci-i386.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 0bd028ccf..bdeadea31 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -58,7 +58,8 @@ jobs: uv run make -j -C zfp/build uv run sudo make -C zfp/build install uv pip install ./zfp - uv pip install --no-build-isolation -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + uv pip install --no-build-isolation -v . + uv pip install "numcodecs[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" shell: alpine.sh {0} - name: List installed packages From 22931c2d883be4e3a3a95b02f3edc716b63b2f92 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:30:45 -0400 Subject: [PATCH 08/23] Don't include pcodec on i386 --- .github/workflows/ci-i386.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index bdeadea31..b23a4a023 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -59,7 +59,7 @@ jobs: uv run sudo make -C zfp/build install uv pip install ./zfp uv pip install --no-build-isolation -v . - uv pip install "numcodecs[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" + uv pip install "numcodecs[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]" shell: alpine.sh {0} - name: List installed packages From eb5750d9687bcfc3d810e72c30a39476a4f9c0e2 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:48:46 -0400 Subject: [PATCH 09/23] Update docs/contributing.rst Co-authored-by: Elliott Sales de Andrade --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f5ddc2f89..1742eb828 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -116,7 +116,7 @@ Setting up with uv (recommended) bootstrap the build dependencies, then sync the project:: $ uv venv - $ uv pip install meson-python meson ninja cython numpy setuptools-scm + $ uv pip install --group dev $ uv sync --group dev --extra test --extra msgpack The first ``uv pip install`` step bootstraps the build tools into the virtualenv. From 454735f401a9512fa852fd3c13999bdd6299b429 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:46:56 -0400 Subject: [PATCH 10/23] Remove unnecessary split --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 863a72081..a3f4d897f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -105,8 +105,7 @@ jobs: - name: Install numcodecs and Zarr run: | - python -m pip install meson-python meson ninja cython numpy setuptools-scm - python -m pip install --no-build-isolation -v ".[test,test_extras]" + python -m pip install -v ".[test,test_extras]" python -m pip install "${{ matrix.zarr-pkg }}" crc32c - name: List installed packages From ddece577a88be706c92db02a7ae7227386a19f5d Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:48:02 -0400 Subject: [PATCH 11/23] Remove unnecessary flag --- src/numcodecs/meson.build | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index ab67a49b6..eca2b2c64 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -2,10 +2,6 @@ c_args = [] link_args = [] -if host_machine.system() == 'darwin' - c_args += ['-stdlib=libc++'] -endif - if host_machine.system() != 'windows' c_args += ['-pthread'] link_args += ['-pthread'] From d470be97781c20e7f4a99a597107db2bfe35b59a Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:50:20 -0400 Subject: [PATCH 12/23] Cleanup gitignore --- .gitignore | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 6966ecfba..ba6821efd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,7 @@ __pycache__/ *.py[cod] *$py.class -# C extensions -src/numcodecs/**/*.c -src/numcodecs/**/*.h +# Shared libraries (legacy, meson builds out-of-source) *.so # editor @@ -50,8 +48,6 @@ coverage.xml .hypothesis/ cover/ -# Cython annotation files -src/numcodecs/*.html # Translations *.mo @@ -103,8 +99,6 @@ ENV/ builddir/ subprojects/ -# Cython generated -src/numcodecs/*.c # pixi environments .pixi/* *.egg-info From e7b12803f6b4c17f221084f27f841ccf78637547 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:52:40 -0400 Subject: [PATCH 13/23] More accurate release notes --- docs/release.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index c571bf590..566d61d2a 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -37,11 +37,11 @@ Maintenance By :user:`Max Jones `. -* Move test suite from ``numcodecs/tests/`` to top-level ``tests/`` directory. This - avoids import shadowing issues with meson-python non-editable installs, where the - source tree's ``numcodecs/`` package (without compiled C extensions) would shadow the - installed package. This is the standard layout for meson-python projects (numpy, scipy, - scikit-learn all use top-level ``tests/``). Test imports change from +* Move source code from ``numcodecs/`` to ``src/numcodecs/`` (src layout). This avoids + import shadowing issues where the source tree's ``numcodecs/`` package (without compiled + C extensions) would shadow the installed package. Move test suite from + ``numcodecs/tests/`` to top-level ``tests/`` directory, following the standard layout + for meson-python projects (numpy, scipy, scikit-learn). Test imports change from ``from numcodecs.tests.common import ...`` to ``from tests.common import ...``. By :user:`Max Jones `. From ec5af111c4ae83e8d01a4a4687c9f0572ac6c6a5 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:54:26 -0400 Subject: [PATCH 14/23] Use dev group --- .github/workflows/ci-i386.yml | 2 +- docs/contributing.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index b23a4a023..6e53d563c 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -48,7 +48,7 @@ jobs: - name: Install zfp and numcodecs run: | uv venv - uv pip install meson-python meson ninja cython numpy setuptools-scm + uv pip install --group dev PYTHON_INCLUDE=$(uv run python -c 'from sysconfig import get_paths; print(get_paths()["include"])'); PYTHON_LIB=$(uv run python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))'); git clone https://github.com/LLNL/zfp diff --git a/docs/contributing.rst b/docs/contributing.rst index 1742eb828..ee9b3037c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -138,7 +138,7 @@ If you prefer the standard library ``venv``:: $ source venv/bin/activate # macOS/Linux $ # .\venv\Scripts\activate # Windows - $ pip install meson-python meson ninja cython numpy setuptools-scm + $ pip install --group dev $ pip install --no-build-isolation -e ".[test,test_extras,msgpack]" $ pytest -v From 794508a4636b1891c5e5c209cbe791b59520c174 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:09:02 -0400 Subject: [PATCH 15/23] Fix contrib instructions --- .gitignore | 4 ---- docs/contributing.rst | 14 +++++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ba6821efd..14facfe79 100644 --- a/.gitignore +++ b/.gitignore @@ -95,10 +95,6 @@ ENV/ # PyCharm .idea -# Meson build artifacts -builddir/ -subprojects/ - # pixi environments .pixi/* *.egg-info diff --git a/docs/contributing.rst b/docs/contributing.rst index ee9b3037c..bd57aef22 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -113,17 +113,17 @@ Setting up with uv (recommended) """"""""""""""""""""""""""""""""" `uv `_ is a fast Python package manager. First, -bootstrap the build dependencies, then sync the project:: +bootstrap the build dependencies, then install numcodecs in editable mode:: $ uv venv $ uv pip install --group dev - $ uv sync --group dev --extra test --extra msgpack + $ uv pip install --no-build-isolation -e ".[test,test_extras,msgpack]" The first ``uv pip install`` step bootstraps the build tools into the virtualenv. -``uv sync`` then builds numcodecs in editable mode (without build isolation, so -the venv's ``ninja`` is used for auto-rebuild on import). This two-step bootstrap -is needed because meson-python editable installs require build tools at runtime, -not just at build time. +The second installs numcodecs in editable mode without build isolation (so the +venv's build tools are used). This two-step process is needed because +meson-python editable installs require build tools at runtime for auto-rebuild +on import. To run the tests:: @@ -167,7 +167,7 @@ Rebuilding after changes With an editable install, meson-python automatically rebuilds changed C/Cython extensions when you import ``numcodecs``. If you need a full clean rebuild:: - $ rm -rf builddir + $ rm -rf build $ pip install --no-build-isolation -e . Creating a branch From 6428dd0d62f73856aa0662be95e9874ccbd3b334 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:14:39 -0400 Subject: [PATCH 16/23] Remove pixi section --- docs/contributing.rst | 20 ++++++++++++++++++++ pyproject.toml | 42 ++---------------------------------------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index bd57aef22..a4bbf197f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -213,6 +213,26 @@ Or, if using venv/pip:: $ pytest -v +To test against specific Zarr-Python versions, use the dependency groups defined in +``pyproject.toml``:: + + $ uv run --group test-zarr-312 pytest tests/test_zarr3.py tests/test_zarr3_import.py + $ uv run --group test-zarr-313 pytest tests/test_zarr3.py tests/test_zarr3_import.py + $ uv run --group test-zarr-main pytest tests/test_zarr3.py tests/test_zarr3_import.py + +Or, if using venv/pip:: + + $ pip install --group test-zarr-312 + $ pytest tests/test_zarr3.py tests/test_zarr3_import.py + +To test with different CRC32C implementations:: + + $ uv pip install "numcodecs[crc32c]" # sw-based crc32c + $ pytest tests/test_checksum32.py -v + + $ uv pip install "numcodecs[google_crc32c]" # hw-accelerated google-crc32c + $ pytest tests/test_checksum32.py -v + All tests are automatically run via GitHub Actions for every pull request across Linux (x86_64, aarch64, i386), macOS (x86_64, arm64), and Windows (x86_64), with Python 3.11 through 3.14. Tests must pass on all platforms before code can be accepted. diff --git a/pyproject.toml b/pyproject.toml index 9f818b30c..4cf166260 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,8 +242,8 @@ warn_unused_configs = true # meson-python editable installs require build deps (ninja, meson) at runtime for # auto-rebuild on import. Skip build isolation so the venv's ninja is used. # The "dev" dependency group provides these build tools. Bootstrap with: -# uv pip install meson-python meson ninja cython numpy setuptools-scm -# before running uv sync. +# `uv pip install --group dev` before running +# `uv pip install --no-build-isolation -e ".[test,test_extras,msgpack]"`. no-build-isolation-package = ["numcodecs"] conflicts = [ # Zarr versions conflict with each other @@ -253,41 +253,3 @@ conflicts = [ { group = "test-zarr-main" } ] ] - -[tool.pixi.workspace] -channels = ["conda-forge"] -platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] - -[tool.pixi.dependencies] -python = "=3.14" -clang = ">=19.1.7,<20" -c-compiler = ">=1.9.0,<2" -cxx-compiler = ">=1.9.0,<2" -uv = "*" - -[tool.pixi.feature.test.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","zfpy"] } - -[tool.pixi.feature.test-google-crc32c.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] } - -[tool.pixi.feature.test-crc32c.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["crc32c"] } - -[tool.pixi.environments] -default = { solve-group = "default" } -test = ["test"] -test-crc32c = ["test", "test-crc32c"] -test-google-crc32c = ["test", "test-google-crc32c"] - -[tool.pixi.tasks] -bootstrap = "uv pip install meson-python meson ninja cython numpy setuptools-scm" -ls-deps-312 = { cmd = "uv run --group test-zarr-312 uv pip freeze", depends-on = ["bootstrap"] } -ls-deps-313 = { cmd = "uv run --group test-zarr-313 uv pip freeze", depends-on = ["bootstrap"] } -ls-deps-main = { cmd = "uv run --group test-zarr-main uv pip freeze", depends-on = ["bootstrap"] } -test-zarr-312 = { cmd = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } -test-zarr-313 = { cmd = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } -test-zarr-main = { cmd = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py", depends-on = ["bootstrap"] } - -[tool.pixi.feature.test.tasks] -run-tests = "pytest -v" From f88d337da7c9453cd6b0923e7a0abb9a7ce400cc Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:22:31 -0400 Subject: [PATCH 17/23] Test backwards compatibility shim --- tests/test_version.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_version.py diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 000000000..a1e30d9a1 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,15 @@ +import numcodecs + + +def test_version(): + assert isinstance(numcodecs.__version__, str) + assert numcodecs.__version__ + + +def test_version_module(): + from numcodecs.version import __version__, version + + assert isinstance(__version__, str) + assert isinstance(version, str) + assert __version__ == version + assert __version__ == numcodecs.__version__ From f1318f57b697a9687919ee67d0024f747a4b2237 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:29:54 -0400 Subject: [PATCH 18/23] Test different crc32c branches --- .github/workflows/ci.yaml | 56 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a3f4d897f..5f98d90bb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: true jobs: - build: + tests: runs-on: ${{ matrix.platform }} strategy: fail-fast: false @@ -69,6 +69,60 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} verbose: true + test-crc32c: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: google-crc32c + extras: "google_crc32c" + - name: crc32c + extras: "crc32c" + - name: no-crc32c + extras: "" + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout source + uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3.2.0 + with: + channels: conda-forge + miniforge-version: latest + python-version: "3.13" + + - name: Install compilers + run: conda install -y c-compiler cxx-compiler + + - name: Install numcodecs + run: | + if [[ -n "${{ matrix.extras }}" ]]; then + python -m pip install -v ".[${{ matrix.extras }},test]" + else + python -m pip install -v ".[test]" + fi + + - name: List installed packages + run: python -m pip list + + - name: Run checksum tests + run: pytest -v tests/test_checksum32.py + + - uses: codecov/codecov-action@v5 + with: + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + test-zarr: runs-on: ubuntu-latest strategy: From 5a522e883bc5bdee9fbe043791ce13b373a7aa3e Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:51:09 -0400 Subject: [PATCH 19/23] Keep old job name --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5f98d90bb..ad21211cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,8 @@ concurrency: cancel-in-progress: true jobs: - tests: + build: + name: Tests runs-on: ${{ matrix.platform }} strategy: fail-fast: false From 762519e0898996d39e512f505de1a613c7bac2d2 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:40:26 -0400 Subject: [PATCH 20/23] Lint --- tests/common.py | 3 ++- tests/test_bitround.py | 1 + tests/test_compat.py | 1 + tests/test_entrypoints.py | 3 ++- tests/test_entrypoints_backport.py | 3 ++- tests/test_ndarray_like.py | 1 + tests/test_registry.py | 3 ++- 7 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/common.py b/tests/common.py index 31685d955..e3c9cfb3e 100644 --- a/tests/common.py +++ b/tests/common.py @@ -6,10 +6,11 @@ import numpy as np import pytest +from numpy.testing import assert_array_almost_equal, assert_array_equal + from numcodecs import * # noqa: F403 # for eval to find names in repr tests from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.registry import get_codec -from numpy.testing import assert_array_almost_equal, assert_array_equal greetings = [ '¡Hola mundo!', diff --git a/tests/test_bitround.py b/tests/test_bitround.py index 663cf73a3..3929e43c6 100644 --- a/tests/test_bitround.py +++ b/tests/test_bitround.py @@ -1,5 +1,6 @@ import numpy as np import pytest + from numcodecs.bitround import BitRound, max_bits # adapted from https://github.com/milankl/BitInformation.jl/blob/main/test/round_nearest.jl diff --git a/tests/test_compat.py b/tests/test_compat.py index 091e34539..0d5f2d74e 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -3,6 +3,7 @@ import numpy as np import pytest + from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_text diff --git a/tests/test_entrypoints.py b/tests/test_entrypoints.py index 25eb9d34d..43503ec39 100644 --- a/tests/test_entrypoints.py +++ b/tests/test_entrypoints.py @@ -1,9 +1,10 @@ import os.path import sys -import numcodecs.registry import pytest +import numcodecs.registry + here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_entrypoints_backport.py b/tests/test_entrypoints_backport.py index ff1b02a67..7e1c32bcc 100644 --- a/tests/test_entrypoints_backport.py +++ b/tests/test_entrypoints_backport.py @@ -3,9 +3,10 @@ import sys from multiprocessing import Process -import numcodecs.registry import pytest +import numcodecs.registry + importlib_spec = importlib.util.find_spec("importlib_metadata") if importlib_spec is None or importlib_spec.loader is None: # pragma: no cover pytest.skip( diff --git a/tests/test_ndarray_like.py b/tests/test_ndarray_like.py index ff10e2400..6c16e7dbf 100644 --- a/tests/test_ndarray_like.py +++ b/tests/test_ndarray_like.py @@ -1,4 +1,5 @@ import pytest + from numcodecs.ndarray_like import DType, FlagsObj, NDArrayLike diff --git a/tests/test_registry.py b/tests/test_registry.py index a5e52ba01..93ed8f032 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,7 +1,8 @@ import inspect -import numcodecs import pytest + +import numcodecs from numcodecs.errors import UnknownCodecError from numcodecs.registry import get_codec From 8147d88861499a179d8b4a78460a2768868b97cd Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:57:37 -0400 Subject: [PATCH 21/23] Collect coverage on all test jobs --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ad21211cc..4597ea605 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,9 +107,9 @@ jobs: - name: Install numcodecs run: | if [[ -n "${{ matrix.extras }}" ]]; then - python -m pip install -v ".[${{ matrix.extras }},test]" + python -m pip install -v -e ".[${{ matrix.extras }},test]" else - python -m pip install -v ".[test]" + python -m pip install -v -e ".[test]" fi - name: List installed packages @@ -160,7 +160,7 @@ jobs: - name: Install numcodecs and Zarr run: | - python -m pip install -v ".[test,test_extras]" + python -m pip install -v -e ".[test,test_extras]" python -m pip install "${{ matrix.zarr-pkg }}" crc32c - name: List installed packages From 22d233ef334fb390eb4f38fa6a978eafd6a3fc85 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 20:56:47 -0400 Subject: [PATCH 22/23] Non-editable install --- .github/workflows/ci.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4597ea605..1abe666dd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,9 +107,9 @@ jobs: - name: Install numcodecs run: | if [[ -n "${{ matrix.extras }}" ]]; then - python -m pip install -v -e ".[${{ matrix.extras }},test]" + python -m pip install -v ".[${{ matrix.extras }},test]" else - python -m pip install -v -e ".[test]" + python -m pip install -v ".[test]" fi - name: List installed packages @@ -160,9 +160,7 @@ jobs: - name: Install numcodecs and Zarr run: | - python -m pip install -v -e ".[test,test_extras]" - python -m pip install "${{ matrix.zarr-pkg }}" crc32c - + python -m pip install -v ".[test,test_extras]" "${{ matrix.zarr-pkg }}" crc32c - name: List installed packages run: python -m pip list From a1c9a430c676effe8e9ed751f217d36fe8b44bd6 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 18 Mar 2026 21:00:34 -0400 Subject: [PATCH 23/23] Fix coverage setting --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1abe666dd..e828c8c7c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -165,7 +165,7 @@ jobs: run: python -m pip list - name: Run Zarr integration tests - run: pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py + run: pytest tests/test_zarr3.py tests/test_zarr3_import.py - uses: codecov/codecov-action@v5 with: