diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 61f46b7c7..6e53d563c 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -45,9 +45,10 @@ jobs: py3-pip py3-pytest - - name: Install zfp + - name: Install zfp and numcodecs run: | uv venv + uv pip install --group dev PYTHON_INCLUDE=$(uv run python -c 'from sysconfig import get_paths; print(get_paths()["include"])'); PYTHON_LIB=$(uv run python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))'); git clone https://github.com/LLNL/zfp @@ -56,19 +57,9 @@ jobs: uv run cmake zfp -B zfp/build -DBUILD_ZFPY=ON -DBUILD_TESTING=OFF -DPYTHON_LIBRARY=$PYTHON_LIB -DPYTHON_INCLUDE_DIR=$PYTHON_INCLUDE uv run make -j -C zfp/build uv run sudo make -C zfp/build install - uv add "zfpy @ ./zfp" - shell: alpine.sh {0} - - - name: Install numcodecs - run: | - export DISABLE_NUMCODECS_AVX2="" - uv venv - # TODO: Remove this conditional when pcodec supports Python 3.14 - if [[ "${{ matrix.python-version }}" == "3.14" ]]; then - uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] - else - uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] - fi + uv pip install ./zfp + uv pip install --no-build-isolation -v . + uv pip install "numcodecs[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]" shell: alpine.sh {0} - name: List installed packages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4030e1948..e828c8c7c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,7 @@ concurrency: jobs: build: + name: Tests runs-on: ${{ matrix.platform }} strategy: fail-fast: false @@ -49,12 +50,11 @@ jobs: - name: Install numcodecs run: | - export DISABLE_NUMCODECS_AVX2="" # TODO: Remove this conditional when pcodec supports Python 3.14 if [[ "${{ matrix.python-version }}" == "3.14" ]]; then - python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,zfpy]" else - python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + python -m pip install -v ".[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy]" fi - name: List installed packages @@ -70,12 +70,72 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} verbose: true + test-crc32c: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: google-crc32c + extras: "google_crc32c" + - name: crc32c + extras: "crc32c" + - name: no-crc32c + extras: "" + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout source + uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3.2.0 + with: + channels: conda-forge + miniforge-version: latest + python-version: "3.13" + + - name: Install compilers + run: conda install -y c-compiler cxx-compiler + + - name: Install numcodecs + run: | + if [[ -n "${{ matrix.extras }}" ]]; then + python -m pip install -v ".[${{ matrix.extras }},test]" + else + python -m pip install -v ".[test]" + fi + + - name: List installed packages + run: python -m pip list + + - name: Run checksum tests + run: pytest -v tests/test_checksum32.py + + - uses: codecov/codecov-action@v5 + with: + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + test-zarr: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - zarr-version: ["312", "313", "main"] + include: + - zarr-version: "312" + zarr-pkg: "zarr==3.1.2" + - zarr-version: "313" + zarr-pkg: "zarr==3.1.3" + - zarr-version: "main" + zarr-pkg: "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main" defaults: run: @@ -88,19 +148,24 @@ jobs: submodules: recursive fetch-depth: 0 # required for version resolution - - name: Set up Pixi - uses: prefix-dev/setup-pixi@v0.9.0 + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3.2.0 with: - pixi-version: v0.49.0 - cache: false + channels: conda-forge + miniforge-version: latest + python-version: "3.13" + + - name: Install compilers + run: conda install -y c-compiler cxx-compiler + - name: Install numcodecs and Zarr + run: | + python -m pip install -v ".[test,test_extras]" "${{ matrix.zarr-pkg }}" crc32c - name: List installed packages - shell: "bash -l {0}" - run: pixi run ls-deps-${{matrix.zarr-version}} + run: python -m pip list - - name: Run tests with Zarr ${{ matrix.zarr-version }} - shell: "bash -l {0}" - run: pixi run test-zarr-${{ matrix.zarr-version }} + - name: Run Zarr integration tests + run: pytest tests/test_zarr3.py tests/test_zarr3_import.py - uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index a006d513d..25e606e21 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -25,7 +25,7 @@ jobs: CIBW_TEST_COMMAND: python -c "import numcodecs" CIBW_BUILD: "cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *win32 *_i686 *_s390x" - # note: CIBW_ENVIRONMENT is now set in pyproject.toml + # note: cibuildwheel config-settings are set in pyproject.toml steps: - uses: actions/checkout@v5 diff --git a/.gitignore b/.gitignore index 5e08553b3..14facfe79 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,7 @@ __pycache__/ *.py[cod] *$py.class -# C extensions -numcodecs/**/*.c -numcodecs/**/*.h +# Shared libraries (legacy, meson builds out-of-source) *.so # editor @@ -50,8 +48,6 @@ coverage.xml .hypothesis/ cover/ -# Cython annotation files -numcodecs/*.html # Translations *.mo @@ -99,12 +95,8 @@ ENV/ # PyCharm .idea -# setuptools-scm -numcodecs/version.py - -# Cython generated -numcodecs/*.c # pixi environments .pixi/* *.egg-info -pixi.lock \ No newline at end of file +pixi.lock +uv.lock \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 6469633e1..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -recursive-include c-blosc * -recursive-include numcodecs *.pyx -recursive-include numcodecs *.pxd diff --git a/adhoc/blosc_memleak_check.py b/adhoc/blosc_memleak_check.py index a6409ed19..427c3e153 100644 --- a/adhoc/blosc_memleak_check.py +++ b/adhoc/blosc_memleak_check.py @@ -1,9 +1,10 @@ import sys -import numcodecs import numpy as np from numpy.testing import assert_array_equal +import numcodecs + codec = numcodecs.Blosc() data = np.arange(int(sys.argv[1])) for _ in range(int(sys.argv[2])): diff --git a/docs/contributing.md b/docs/contributing.md index 19a0d8227..c9f495586 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -31,16 +31,9 @@ a bug report: Python interpreter, and installation information. The version of NumCodecs can be obtained from the `numcodecs.__version__` property. Please also state how NumCodecs was installed, e.g., "installed via pip into a virtual environment", or "installed using conda". - Information about other packages installed can be obtained by executing `pip list` - (if using pip to install packages) or `conda list` (if using conda to install - packages) from the operating system command prompt. The version of the Python - interpreter can be obtained by running a Python interactive session, e.g.: - - ``` - $ python - Python 3.8.15 | packaged by conda-forge | (default, Nov 22 2022, 08:49:06) - [Clang 14.0.6 ] on darwin - ``` + Information about other packages installed can be obtained by executing ``pip list`` + (if using pip to install packages) or ``conda list`` (if using conda to install + packages) from the operating system command prompt. 3. An explanation of why the current behaviour is wrong/not desired, and what you expect instead. @@ -85,34 +78,106 @@ $ cd numcodecs $ git remote add upstream https://github.com/zarr-developers/numcodecs.git ``` +Note the ``--recursive`` flag is required to clone the ``c-blosc`` git submodule. If you +forgot it, you can initialize it later with: + +``` +$ git submodule update --init --recursive +``` + ### Creating a development environment -To work with the NumCodecs source code, it is recommended to set up a Python virtual -environment and install all NumCodecs dependencies using the same versions as are used by -the core developers and continuous integration services. Assuming you have a Python -3 interpreter already installed matching the `requires-python` constraint from -`pyproject.toml`, and you have cloned the NumCodecs source code and your -current working directory is the root of the repository, you can do something -like the following: +NumCodecs contains C and Cython extensions, so you need a C compiler and build +tooling in addition to Python. + +#### Prerequisites + +You need a C compiler available on your ``PATH``. + +On Debian/Ubuntu: ``` -$ python3 -m venv ~/pyenv/numcodecs-dev -$ source ~/pyenv/numcodecs-dev/bin/activate -$ pip install -e .[docs,test,msgpack,zfpy] +$ sudo apt install build-essential ``` -You may need to initialize the submodule for c-blosc: +On macOS (Xcode command line tools): ``` -$ git submodule update --init --recursive +$ xcode-select --install +``` + +On Windows, install [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) with the "Desktop development +with C++" workload. + +#### Setting up with uv (recommended) + +[uv](https://docs.astral.sh/uv/) is a fast Python package manager. First, +bootstrap the build dependencies, then install numcodecs in editable mode: + +``` +$ uv venv +$ uv pip install --group dev +$ uv pip install --no-build-isolation -e ".[test,test_extras,msgpack]" +``` + +The first ``uv pip install`` step bootstraps the build tools into the virtualenv. +The second installs numcodecs in editable mode without build isolation (so the +venv's build tools are used). This two-step process is needed because +meson-python editable installs require build tools at runtime for auto-rebuild +on import. + +To run the tests: + +``` +$ uv run pytest -v ``` -To verify that your development environment is working, you can run the unit tests: +#### Setting up with venv and pip + +If you prefer the standard library ``venv``: ``` +$ python -m venv venv +$ source venv/bin/activate # macOS/Linux +$ # .\venv\Scripts\activate # Windows + +$ pip install --group dev +$ pip install --no-build-isolation -e ".[test,test_extras,msgpack]" $ pytest -v ``` +#### Passing build options + +NumCodecs uses [Meson](https://mesonbuild.com) as its build system. You can +pass Meson options via pip's ``--config-settings``. + +To build against system-installed libraries instead of the vendored copies: + +``` +$ pip install --no-build-isolation -e . \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled +``` + +To disable SIMD optimizations (e.g., for portable debugging): + +``` +$ pip install --no-build-isolation -e . \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled +``` + +#### Rebuilding after changes + +With an editable install, meson-python automatically rebuilds changed C/Cython +extensions when you import ``numcodecs``. If you need a full clean rebuild: + +``` +$ rm -rf build +$ pip install --no-build-isolation -e . +``` + ### Creating a branch Before you do any new work or submit a pull request, please open an issue on GitHub to @@ -152,18 +217,47 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite NumCodecs includes a suite of unit tests, as well as doctests included in function and class -docstrings. The simplest way to run the unit tests is to invoke: +docstrings: + +``` +$ uv run pytest -v +``` + +Or, if using venv/pip: ``` $ pytest -v ``` -NumCodecs currently supports Python 6-3.9, so the above command must -succeed before code can be accepted into the main code base. +To test against specific Zarr-Python versions, use the dependency groups defined in +``pyproject.toml``: + +``` +$ uv run --group test-zarr-312 pytest tests/test_zarr3.py tests/test_zarr3_import.py +$ uv run --group test-zarr-313 pytest tests/test_zarr3.py tests/test_zarr3_import.py +$ uv run --group test-zarr-main pytest tests/test_zarr3.py tests/test_zarr3_import.py +``` + +Or, if using venv/pip: + +``` +$ pip install --group test-zarr-312 +$ pytest tests/test_zarr3.py tests/test_zarr3_import.py +``` + +To test with different CRC32C implementations: + +``` +$ uv pip install "numcodecs[crc32c]" # sw-based crc32c +$ pytest tests/test_checksum32.py -v + +$ uv pip install "numcodecs[google_crc32c]" # hw-accelerated google-crc32c +$ pytest tests/test_checksum32.py -v +``` -All tests are automatically run via Travis (Linux) and AppVeyor (Windows) continuous -integration services for every pull request. Tests must pass under both services before -code can be accepted. +All tests are automatically run via GitHub Actions for every pull request across Linux +(x86_64, aarch64, i386), macOS (x86_64, arm64), and Windows (x86_64), with Python 3.11 +through 3.14. Tests must pass on all platforms before code can be accepted. ### Code standards @@ -177,11 +271,10 @@ $ pre-commit run ruff ### Test coverage -NumCodecs maintains 100% test coverage under the latest Python stable release (currently -Python 3.9). Both unit tests and docstring doctests are included when computing -coverage. Running `pytest -v` will automatically run the test suite with coverage -and produce a coverage report. This should be 100% before code can be accepted into the -main code base. +NumCodecs maintains 100% test coverage under the latest stable Python release. Both unit +tests and docstring doctests are included when computing coverage. Running ``pytest -v`` +will automatically run the test suite with coverage and produce a coverage report. This +should be 100% before code can be accepted into the main code base. When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull @@ -190,8 +283,7 @@ request. Codecov coverage must also be 100% before code can be accepted. ### Documentation Docstrings for user-facing classes and functions should follow the [numpydoc](https://numpydoc.readthedocs.io/en/latest/format.html#docstring-standard) standard, -including sections for Parameters and Examples. All examples will be run as doctests -under Python 3.9. +including sections for Parameters and Examples. All examples will be run as doctests. NumCodecs uses Sphinx for documentation, hosted on readthedocs.org. Documentation is written in Markdown (.md files) using [MyST](https://myst-parser.readthedocs.io/) in the `docs` folder. diff --git a/docs/design/meson-python-migration.md b/docs/design/meson-python-migration.md new file mode 100644 index 000000000..1083e2445 --- /dev/null +++ b/docs/design/meson-python-migration.md @@ -0,0 +1,656 @@ +# Design Doc: Migrate numcodecs build system from setuptools/setup.py to meson-python + +**Status:** Proposal +**Date:** 2026-03-17 +**Author:** Max (with Claude) + +## Motivation + +numcodecs currently uses a hybrid build system: `pyproject.toml` declares the setuptools backend, but a 386-line `setup.py` contains all the C/Cython extension logic. This is problematic because: + +1. **`setup.py` is effectively deprecated.** PEP 517/518 moved the ecosystem toward declarative builds. `setup.py` is imperative, opaque to tooling, and executed during metadata extraction. +2. **`distutils` imports are fragile.** `setup.py` imports from `distutils` (removed from stdlib in Python 3.12). This only works because setuptools vendors a copy, which may be dropped in a future major release. +3. **CPU feature detection is incorrect for cross-compilation.** `py-cpuinfo` runs at build time on the *build* machine, not the *target* machine. This means cibuildwheel arm64 builds on x86_64 hosts detect x86_64 features. The current workaround is environment variables (`DISABLE_NUMCODECS_AVX2`, `DISABLE_NUMCODECS_SSE2`). +4. **System library linking is unsupported.** PR [#569](https://github.com/zarr-developers/numcodecs/pull/569) has been open since 2023, attempting to add `pkg-config`-based system library support. The approach adds complexity to an already complex `setup.py`. +5. **The scientific Python ecosystem has standardized on meson-python.** numpy, scipy, scikit-learn, scikit-image, and matplotlib have all migrated. Meson natively handles every concern currently implemented manually in `setup.py`. + +## Proposal + +Replace `setup.py` + setuptools with `meson.build` + `meson-python` as the build backend. Remove `py-cpuinfo` as a build dependency. Use meson's native compiler introspection for CPU feature detection and `dependency()` for optional system library linking. + +## Current Architecture + +### Build dependencies + +```toml +[build-system] +requires = ["setuptools>=77", "setuptools-scm[toml]>=6.2", "Cython", "py-cpuinfo", "numpy>2"] +build-backend = "setuptools.build_meta" +``` + +### Extension modules (8 total) + +| Extension | Cython source | C library deps | Assembly | Notes | +|-----------|--------------|----------------|----------|-------| +| `blosc` | `blosc.pyx` | c-blosc + vendored lz4, zlib, zstd | zstd `*amd64.S` / `*aarch64.S` | Most complex; SSE2/AVX2 conditional | +| `zstd` | `zstd.pyx` | vendored zstd (from c-blosc) | zstd `*amd64.S` / `*aarch64.S` | Shares zstd sources with blosc | +| `lz4` | `lz4.pyx` | vendored lz4 (from c-blosc) | None | | +| `vlen` | `vlen.pyx` | None | None | Needs numpy include dirs | +| `fletcher32` | `fletcher32.pyx` | None | None | | +| `jenkins` | `jenkins.pyx` | None | None | Has `CYTHON_TRACE=1` (likely accidental) | +| `compat_ext` | `compat_ext.pyx` | None | None | | +| `_shuffle` | `_shuffle.pyx` | None | None | | + +### Shared Cython declarations + +- `compat_ext.pxd` — `PyBytes_RESIZE`, `ensure_continguous_memoryview` (used by blosc, zstd, lz4, vlen) +- `_utils.pxd` — `store_le32`, `load_le32` (used by lz4, fletcher32, vlen) + +### Vendored C sources (c-blosc git submodule) + +``` +c-blosc/ +├── blosc/ # Core blosc library (28 files, incl. SIMD shuffle variants) +└── internal-complibs/ + ├── lz4-1.10.0/ # 4 files + ├── zlib-1.3.1/ # ~58 files + └── zstd-1.5.6/ # ~90 files across common/, compress/, decompress/, dictBuilder/ +``` + +### Conditional compilation logic in setup.py + +1. **CPU detection:** `py-cpuinfo` reads `/proc/cpuinfo` or equivalent → sets `have_sse2`, `have_avx2` +2. **Environment overrides:** `DISABLE_NUMCODECS_SSE2`, `DISABLE_NUMCODECS_AVX2`, `DISABLE_NUMCODECS_CEXT` +3. **SIMD sources:** SSE2/AVX2 shuffle/bitshuffle `.c` files included conditionally +4. **SIMD defines:** `-DSHUFFLE_SSE2_ENABLED`, `-DSHUFFLE_AVX2_ENABLED`, `__SSE2__`, `__AVX2__` +5. **Assembly:** `.S` files pre-compiled to `.o`, passed as `extra_objects` +6. **Platform flags:** `-stdlib=libc++` (macOS), `-pthread` (POSIX), `-Wno-implicit-function-declaration` (cibuildwheel macOS) + +## Proposed Architecture + +### Build dependencies + +```toml +[build-system] +requires = ["meson-python>=0.17", "meson>=1.6.0", "Cython>=3.0", "numpy>=2"] +build-backend = "mesonpy" +``` + +**Removed:** `setuptools`, `setuptools-scm`, `py-cpuinfo` + +### File structure + +``` +numcodecs/ +├── meson.build # Root: project(), subdir() +├── meson.options # Build options (system libs, SIMD toggles) +├── numcodecs/ +│ └── meson.build # Extension modules +├── c-blosc/ # Unchanged submodule +├── pyproject.toml # Updated build-backend +└── setup.py # DELETED +``` + +### Root meson.build + +```meson +project( + 'numcodecs', + 'c', 'cython', + version: run_command('python', '-m', 'setuptools_scm', '--version', check: true).stdout().strip(), + default_options: [ + 'c_std=c11', + 'warning_level=1', + ], + meson_version: '>=1.6.0', +) + +py = import('python').find_installation(pure: false) +cy = meson.get_compiler('cython') +cc = meson.get_compiler('c') + +# NumPy include directory (needed for vlen extension) +numpy_dep = dependency('numpy') + +subdir('numcodecs') +``` + +**Note on versioning:** meson-python has native setuptools-scm integration — see [meson-python docs on dynamic versioning](https://mesonbuild.com/meson-python/how-to-guides/dynamic-version.html). The `version` field can use `run_command` to call setuptools-scm, or we can switch to `meson-python`'s built-in VCS versioning. This will need to replace the current `write_to = "numcodecs/version.py"` approach — meson-python generates version metadata at build time via importlib.metadata instead of writing a file. + +### meson.options + +```meson +option('system_blosc', type: 'feature', value: 'auto', + description: 'Use system-installed Blosc library') +option('system_zstd', type: 'feature', value: 'auto', + description: 'Use system-installed Zstandard library') +option('system_lz4', type: 'feature', value: 'auto', + description: 'Use system-installed LZ4 library') +``` + +With `value: 'disabled'`, meson always uses the vendored sources by default, matching the old setup.py behavior. Users who want to link against system libraries opt in with `-Dsystem_blosc=enabled` etc. This replaces the `NUMCODECS_USE_SYSTEM_LIBS` env var from PR #569 with a standard meson idiom. + +### numcodecs/meson.build (core of the migration) + +This is the most complex file. The design below is broken into sections. + +#### Compiler flags + +```meson +# Platform-specific flags +c_args = [] +link_args = [] + +if host_machine.system() == 'darwin' + c_args += ['-stdlib=libc++'] +endif + +if host_machine.system() != 'windows' + c_args += ['-pthread'] + link_args += ['-pthread'] +endif +``` + +#### SIMD detection (replaces py-cpuinfo) + +```meson +# SIMD support — detected via compiler capability, NOT runtime CPU flags. +# This is correct for cross-compilation: we check what the *target* supports. +have_sse2 = false +have_avx2 = false + +if host_machine.cpu_family() == 'x86_64' + have_sse2 = cc.has_argument('-msse2') + have_avx2 = cc.has_argument('-mavx2') +endif + +# Allow disabling via meson configure (replaces DISABLE_NUMCODECS_* env vars) +# cibuildwheel can pass -Dsse2=disabled instead of DISABLE_NUMCODECS_SSE2=1 +``` + +This fixes the cross-compilation bug: `host_machine.cpu_family()` reflects the *target* architecture (from a meson cross file or native detection), not the build machine. + +#### Vendored libraries as static dependencies + +```meson +# --- Vendored zstd --- +zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + +if not zstd_dep.found() + zstd_sources = files( + # common/ + 'c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + 'c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + # ... (enumerate all .c files) + ) + + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files('c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S') + elif host_machine.cpu_family() == 'aarch64' + zstd_sources += files('c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_aarch64.S') + endif + + zstd_inc = include_directories( + 'c-blosc/internal-complibs/zstd-1.5.6', + 'c-blosc/internal-complibs/zstd-1.5.6/common', + 'c-blosc/internal-complibs/zstd-1.5.6/compress', + 'c-blosc/internal-complibs/zstd-1.5.6/decompress', + 'c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', + ) + + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, + c_args: c_args, + ) + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, + ) +endif + +# --- Vendored lz4 --- +lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + +if not lz4_dep.found() + lz4_sources = files( + 'c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + 'c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('c-blosc/internal-complibs/lz4-1.10.0') + + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, + ) +endif + +# --- Vendored zlib --- +zlib_sources = files( + 'c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + 'c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + 'c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + 'c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + 'c-blosc/internal-complibs/zlib-1.3.1/trees.c', + 'c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + 'c-blosc/internal-complibs/zlib-1.3.1/zutil.c', +) +zlib_inc = include_directories('c-blosc/internal-complibs/zlib-1.3.1') + +zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, +) +zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, +) +``` + +#### Blosc extension (most complex) + +```meson +# --- Vendored blosc --- +blosc_dep = dependency('blosc', required: get_option('system_blosc')) + +if not blosc_dep.found() + blosc_sources = files( + 'c-blosc/blosc/blosc.c', + 'c-blosc/blosc/blosclz.c', + 'c-blosc/blosc/fastcopy.c', + 'c-blosc/blosc/shuffle.c', + 'c-blosc/blosc/shuffle-generic.c', + 'c-blosc/blosc/bitshuffle-generic.c', + ) + + blosc_c_args = c_args + blosc_c_args += ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + 'c-blosc/blosc/shuffle-sse2.c', + 'c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif + endif + + if have_avx2 + blosc_sources += files( + 'c-blosc/blosc/shuffle-avx2.c', + 'c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif + + blosc_inc = include_directories('c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) +endif +``` + +#### Extension module declarations + +```meson +# --- Cython extension modules --- + +# Extensions with C library dependencies +py.extension_module('blosc', + 'blosc.pyx', + dependencies: [blosc_dep, lz4_dep, zlib_dep, zstd_dep], + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('zstd', + 'zstd.pyx', + dependencies: [zstd_dep], + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('lz4', + 'lz4.pyx', + dependencies: [lz4_dep], + include_directories: include_directories('.'), + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('vlen', + 'vlen.pyx', + dependencies: [numpy_dep], + include_directories: include_directories('.'), + install: true, + subdir: 'numcodecs', +) + +# Pure-Cython extensions (no C library deps) +foreach ext : ['fletcher32', 'jenkins', 'compat_ext', '_shuffle'] + py.extension_module(ext, + ext + '.pyx', + include_directories: include_directories('.'), + install: true, + subdir: 'numcodecs', + ) +endforeach + +# --- Pure Python sources --- +py.install_sources( + '__init__.py', + 'abc.py', + 'compat.py', + 'registry.py', + # ... all .py files + subdir: 'numcodecs', +) +``` + +### Version management + +**Option A: meson-python + setuptools-scm (minimal change)** + +Keep `setuptools-scm` as a build dependency and use `run_command()` in `meson.build`: + +```meson +project('numcodecs', 'c', 'cython', + version: run_command( + py, ['-m', 'setuptools_scm', '--version'], check: true + ).stdout().strip(), +) +``` + +At runtime, version is read via `importlib.metadata.version('numcodecs')` instead of a generated `version.py`. Update `__init__.py`: + +```python +from importlib.metadata import version +__version__ = version("numcodecs") +``` + +**Option B: meson's built-in VCS tagging** + +Use `vcs_tag()` to generate a version file from git tags, removing the setuptools-scm dependency entirely. This is what numpy does. + +**Recommendation:** Option A — keep setuptools-scm for now, swap to `vcs_tag()` later if desired. + +### pyproject.toml changes + +```toml +[build-system] +requires = ["meson-python>=0.17", "meson>=1.6.0", "Cython>=3.0", "numpy>=2"] +build-backend = "mesonpy" + +[project] +# ... unchanged ... + +[tool.meson-python.args] +# Pass default meson options +setup = ['--default-library=static'] + +# Replace setuptools-scm config +[tool.setuptools_scm] # REMOVE this section + +# Replace setuptools config +[tool.setuptools] # REMOVE this section +``` + +### cibuildwheel changes + +The `DISABLE_NUMCODECS_*` environment variables are replaced by meson configure options: + +```toml +[tool.cibuildwheel] +# Meson cross-compilation handles SIMD correctly — no env vars needed +# AVX2 still disabled for portable wheels (not all x86_64 CPUs have it) +config-settings = "setup-args=-Davx2=disabled" + +[tool.cibuildwheel.macos] +# No need for MACOSX_DEPLOYMENT_TARGET — meson-python handles this +# No need for -Wno-implicit-function-declaration — meson compiles correctly + +[[tool.cibuildwheel.overrides]] +select = "*-macosx_arm64" +config-settings = "setup-args=-Davx2=disabled -Dsse2=disabled" +``` + +### Files deleted + +- `setup.py` (386 lines) +- `MANIFEST.in` (meson-python builds sdists from version control, like setuptools-scm) +- `numcodecs/version.py` (replaced by `importlib.metadata`) + +### Files added + +- `meson.build` (root, ~20 lines) +- `meson.options` (~15 lines) +- `numcodecs/meson.build` (~180 lines) + +### Files updated + +- `docs/contributing.rst` (rewritten — see [Contributing Guide Updates](#contributing-guide-updates) below) + +## Migration Plan + +numcodecs has limited maintenance bandwidth. A phased migration (the numpy/scipy approach of running two build systems in parallel for multiple releases) would double the maintenance surface during the transition. Instead, this should be a single PR that swaps everything at once. + +### Single-PR migration + +One PR, one release. The PR: + +1. **Adds** `meson.build`, `meson.options`, `numcodecs/meson.build` +2. **Deletes** `setup.py`, `MANIFEST.in` +3. **Updates** `pyproject.toml`: new `[build-system]`, removes `[tool.setuptools]`, `[tool.setuptools_scm]`, `[tool.setuptools.package-data]` +4. **Updates** `numcodecs/__init__.py`: version from `importlib.metadata` +5. **Updates** cibuildwheel config in `pyproject.toml` +6. **Updates** CI workflows (pixi tasks, wheel building) +7. **Updates** `.gitignore` (remove `numcodecs/version.py`) +8. **Updates** `docs/contributing.rst`: new dev setup instructions for pixi and uv, removes outdated references +9. **Closes** PR #569 (system library support is native via `-Dsystem_blosc=enabled`) + +### Validation checklist + +Before merging, verify: + +- [ ] `pip install .` works (non-editable) +- [ ] `pip install -e . --no-build-isolation` works (editable) +- [ ] `python -m build --sdist` produces correct sdist (includes c-blosc submodule) +- [ ] `python -m build --wheel` produces working wheel +- [ ] cibuildwheel produces wheels for all target platforms +- [ ] All tests pass on Linux x86_64, Linux aarch64, macOS arm64, macOS x86_64, Windows x86_64 +- [ ] i386 Alpine CI still works (may need a meson cross file) +- [ ] `pixi run run-tests` works for local development +- [ ] `-Dsystem_blosc=enabled` links against system blosc (replaces PR #569) +- [ ] Version string is correct in built package (`python -c "import numcodecs; print(numcodecs.__version__)"`) + +### Risk of a clean break + +The main risk is that a broken build blocks the next release. This is mitigated by: + +- **The test matrix is comprehensive.** CI already tests 5 OS/arch combos x 4 Python versions. If the meson build passes CI, it works. +- **meson-python is battle-tested.** numpy, scipy, scikit-learn all use it for projects with far more complex C/Cython builds than numcodecs. +- **Rollback is trivial.** If something goes wrong post-release, revert the PR and cut a patch release. The old `setup.py` is in git history. + +## Contributing Guide Updates + +The current `docs/contributing.rst` is significantly outdated (references Python 3.8/3.9, Travis CI, AppVeyor, bare `pip install -e .` without a venv manager). The meson migration is a natural point to rewrite the development setup section. + +### Current problems with the contributing guide + +- Recommends manual `python3 -m venv` + `pip install -e .` — works but requires the user to have a C compiler, Cython, and numpy already available, with no guidance on obtaining them +- No mention of pixi or uv, which are already configured in the project +- References Python 3.8, 3.9, Travis CI, and AppVeyor (all outdated) +- No mention of meson or how to pass build options +- No guidance on building against system libraries + +### Proposed "Creating a development environment" section + +The guide should present two supported paths (pixi and uv) and explicitly discourage developing without an isolated environment. The following replaces the "Creating a development environment" and "Running the test suite" sections. + +```rst +Creating a development environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NumCodecs contains C and Cython extensions, so you need a C compiler and +build tooling in addition to Python. We support two development workflows: +**pixi** (recommended) and **uv**. Both manage isolated environments and +are tested in CI. **Do not** install numcodecs into your system Python or a +manually-managed virtualenv — the C compilation dependencies are +difficult to get right by hand. + +Whichever tool you use, first clone the repository with its submodules:: + + $ git clone --recursive git@github.com:your-user-name/numcodecs.git + $ cd numcodecs + $ git remote add upstream https://github.com/zarr-developers/numcodecs.git + +Using pixi (recommended) +""""""""""""""""""""""""" + +`pixi `_ manages Python, the C compiler toolchain, and all +dependencies via conda-forge. This is the easiest way to get started, +especially on macOS or if you don't have a system C compiler. + +Install pixi, then:: + + $ pixi run -e test-py313 run-tests # build + test in one step + +This will create an isolated environment with Python 3.13, install the C +toolchain from conda-forge, build the Cython extensions, and run the test +suite. Available test environments:: + + test-py311, test-py312, test-py313, test-py314 + +To get a shell inside a pixi environment for interactive development:: + + $ pixi shell -e test-py313 + +Using uv +"""""""" + +`uv `_ is a fast Python package manager. It +manages virtualenvs and dependencies but does **not** provide a C compiler — +you must have ``cc`` / ``gcc`` / ``clang`` available on your ``PATH``. + +On Debian/Ubuntu:: + + $ sudo apt install build-essential + +On macOS (Xcode command line tools):: + + $ xcode-select --install + +Then:: + + $ uv sync --extra test --extra msgpack + $ uv run pytest -v + +uv will create a ``.venv``, build the extensions, and install numcodecs in +editable mode. + +Passing build options +""""""""""""""""""""" + +NumCodecs uses `meson `_ as its build system. You can +pass meson options via pip's ``--config-settings`` or via environment variables +when building. + +To build against system-installed Blosc, Zstd, and LZ4 instead of the +vendored copies:: + + $ pip install -e . --no-build-isolation \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled + +To disable SIMD optimizations (e.g. for portable debugging):: + + $ pip install -e . --no-build-isolation \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled + +Running the test suite +~~~~~~~~~~~~~~~~~~~~~~ + +With pixi:: + + $ pixi run -e test-py313 run-tests + +With uv:: + + $ uv run pytest -v + +To run tests for specific Zarr integration versions:: + + $ pixi run test-zarr-312 + $ pixi run test-zarr-313 + $ pixi run test-zarr-main + +All tests are automatically run via GitHub Actions for every pull request +across Linux (x86_64, aarch64, i386), macOS (x86_64, arm64), and Windows +(x86_64), with Python 3.11 through 3.14. +``` + +### Other sections to update + +The rest of the contributing guide needs lighter touch-ups as part of this PR: + +- **"Code standards"** — already correct (references `pre-commit run ruff`), no change needed +- **"Test coverage"** — update Python version reference from "3.9" to "3.13" (or just say "the latest stable release") +- **"Documentation"** — update Python version reference, otherwise fine +- **"Bug reports"** — update the Python session example from 3.8 to a current version +- **"Running the test suite" (old)** — replace entirely (covered above) +- Remove all references to Travis CI and AppVeyor — CI is GitHub Actions only + +## Risks and Mitigations + +### Risk: Meson doesn't support a platform/arch that setuptools did + +**Mitigation:** Meson supports all platforms numcodecs targets (Linux x86_64/aarch64/i386, macOS x86_64/arm64, Windows x86_64). The i386 Alpine CI job (`ci-i386.yml`) needs verification — meson may need a cross file for 32-bit builds inside the Alpine container. + +### Risk: Cython version incompatibilities + +**Mitigation:** Pin `Cython>=3.0` in build-requires. Meson's Cython support requires Cython 3. This drops Cython 0.x support, which is already effectively unsupported. + +### Risk: sdist contents change + +**Mitigation:** meson-python includes all files tracked by git (plus submodules) in sdists by default, matching setuptools-scm behavior. The `MANIFEST.in` is not needed. Verify with `python -m build --sdist` and compare contents. + +### Risk: Editable installs behave differently + +**Mitigation:** meson-python supports editable installs via `pip install -e . --no-build-isolation`. The experience differs from setuptools (meson-python uses import hooks rather than `.egg-link`). The pixi-based dev workflow should still work. Test editable installs explicitly in Phase 1. + +### Risk: Windows MSVC compatibility + +**Mitigation:** Meson has excellent MSVC support. The SIMD flag logic (`-msse2`, `-mavx2`) is GCC/Clang-specific; MSVC equivalents (`/arch:SSE2`, `/arch:AVX2`) are needed. Meson's `cc.get_id()` can branch on compiler. Current `setup.py` already handles this partially (the `__SSE2__`/`__AVX2__` defines for `os.name == 'nt'`). + +## References + +- [meson-python documentation](https://mesonbuild.com/meson-python/) +- [NumPy meson migration](https://numpy.org/doc/stable/reference/distutils_status_migration.html) +- [SciPy meson migration PR](https://github.com/scipy/scipy/pull/15959) +- [Meson Cython support](https://mesonbuild.com/Cython.html) +- [Meson SIMD module](https://mesonbuild.com/Simd-module.html) +- [PR #569 — system library linking](https://github.com/zarr-developers/numcodecs/pull/569) +- [Issue #464 — use system libraries](https://github.com/zarr-developers/numcodecs/issues/464) diff --git a/docs/index.md b/docs/index.md index e6808a86c..bdaf1c8c2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,16 +37,25 @@ $ pip install -v --no-cache-dir --no-binary numcodecs numcodecs Note that if you compile the C extensions on a machine with AVX2 support you probably then cannot use the same binaries on a machine without AVX2. -If you specifically want to disable AVX2 or SSE2 when compiling, you can use -the following environment variables: +If you specifically want to disable AVX2 or SSE2 when compiling from source, +you can pass meson build options:: + + $ pip install -v --no-cache-dir --no-binary numcodecs numcodecs \ + --config-settings=setup-args=-Davx2=disabled \ + --config-settings=setup-args=-Dsse2=disabled + +You can also build against system-installed Blosc, Zstd, and LZ4 libraries +instead of the vendored copies:: + + $ pip install -v --no-cache-dir --no-binary numcodecs numcodecs \ + --config-settings=setup-args=-Dsystem_blosc=enabled \ + --config-settings=setup-args=-Dsystem_zstd=enabled \ + --config-settings=setup-args=-Dsystem_lz4=enabled -``` -$ export DISABLE_NUMCODECS_AVX2=1 -$ export DISABLE_NUMCODECS_SSE2=1 -``` -To work with Numcodecs source code in development, clone the repository from GitHub -and then install in editable mode using `pip`. +To work with Numcodecs source code in development, see the +`contributing guide `_ for instructions on setting up a +development environment with venv or uv. ``` $ git clone --recursive https://github.com/zarr-developers/numcodecs.git diff --git a/docs/release.md b/docs/release.md index e2fb936a2..78e8901c5 100644 --- a/docs/release.md +++ b/docs/release.md @@ -16,6 +16,33 @@ ### Maintenance +* **Migrate build system from setuptools/setup.py to meson-python.** This replaces the + 386-line ``setup.py`` with declarative ``meson.build`` files. Benefits include correct + SIMD detection for cross-compilation, native support for linking against system-installed + Blosc/Zstd/LZ4 libraries (via ``-Dsystem_blosc=enabled`` etc.), and alignment with the + build system used by numpy, scipy, and scikit-learn. + + The ``DISABLE_NUMCODECS_AVX2`` and ``DISABLE_NUMCODECS_SSE2`` environment variables + continue to work for backwards compatibility. The preferred way to control SIMD is now + via meson options:: + + pip install numcodecs --no-binary numcodecs \ + --config-settings=setup-args=-Davx2=disabled + + The ``DISABLE_NUMCODECS_CEXT`` environment variable is no longer supported. + + By :user:`Max Jones `. + +* Move source code from ``numcodecs/`` to ``src/numcodecs/`` (src layout). This avoids + import shadowing issues where the source tree's ``numcodecs/`` package (without compiled + C extensions) would shadow the installed package. + + By :user:`Max Jones `. + +* Rewrite contributing guide with uv development environment instructions. + + By :user:`Max Jones `. + * Convert documentation from reStructuredText to Markdown using MyST. By {user}`Max Jones `, :issue:`830` diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..8d90d9c78 --- /dev/null +++ b/meson.build @@ -0,0 +1,20 @@ +project( + 'numcodecs', + 'c', 'cython', + version: run_command( + ['src/numcodecs/_build_utils/gitversion.py'], check: true, + ).stdout().strip(), + default_options: [ + 'c_std=gnu11', + 'warning_level=1', + ], + meson_version: '>=1.6.0', +) + +py = import('python').find_installation(pure: false) +cc = meson.get_compiler('c') + +# NumPy include directory (needed for vlen extension) +numpy_dep = dependency('numpy') + +subdir('src/numcodecs') diff --git a/meson.options b/meson.options new file mode 100644 index 000000000..26a2414d6 --- /dev/null +++ b/meson.options @@ -0,0 +1,10 @@ +option('system_blosc', type: 'feature', value: 'disabled', + description: 'Use system-installed Blosc library') +option('system_zstd', type: 'feature', value: 'disabled', + description: 'Use system-installed Zstandard library') +option('system_lz4', type: 'feature', value: 'disabled', + description: 'Use system-installed LZ4 library') +option('sse2', type: 'feature', value: 'auto', + description: 'Enable SSE2 SIMD optimizations') +option('avx2', type: 'feature', value: 'auto', + description: 'Enable AVX2 SIMD optimizations') diff --git a/notebooks/benchmark_vlen.ipynb b/notebooks/benchmark_vlen.ipynb index 6870bef6a..cebaf7f00 100644 --- a/notebooks/benchmark_vlen.ipynb +++ b/notebooks/benchmark_vlen.ipynb @@ -24,9 +24,10 @@ } ], "source": [ - "import numcodecs\n", "import numpy as np\n", "\n", + "import numcodecs\n", + "\n", "numcodecs.__version__" ] }, diff --git a/pyproject.toml b/pyproject.toml index e1e93e8e9..50353191c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [build-system] requires = [ - "setuptools>=77", - "setuptools-scm[toml]>=6.2", - "Cython", - "py-cpuinfo", - "numpy>2", + "meson-python>=0.17", + "meson>=1.6.0", + "setuptools-scm>=6.2", + "Cython>=3.0", + "numpy>=2", ] -build-backend = "setuptools.build_meta" +build-backend = "mesonpy" [project] name = "numcodecs" @@ -81,6 +81,14 @@ test_extras = [ ] [dependency-groups] +dev = [ + "meson-python>=0.17", + "meson>=1.6.0", + "ninja", + "cython>=3.0", + "setuptools-scm>=6.2", + "numpy>=2", +] test-zarr-312 = [ "pytest", "pytest-cov", @@ -100,19 +108,6 @@ test-zarr-main = [ "crc32c", ] -[tool.setuptools] -package-dir = {"" = "."} -packages = ["numcodecs"] -zip-safe = false - -[tool.setuptools.package-data] -numcodecs = [] - -[tool.setuptools_scm] -version_scheme = "guess-next-dev" -local_scheme = "dirty-tag" -write_to = "numcodecs/version.py" - [tool.codespell] skip = "./.git,fixture" ignore-words-list = "ba, compiletime, hist, nd, unparseable" @@ -145,6 +140,8 @@ norecursedirs = [ ".pytest_cache", "adhoc", "build", + "builddir", + "subprojects", "c-blosc", "docs", "fixture", @@ -159,19 +156,16 @@ filterwarnings = [ ] [tool.cibuildwheel] -environment = { DISABLE_NUMCODECS_AVX2=1 } -[tool.cibuildwheel.macos] -# cibuildwheel uses 3.12 for the Python driver, which supports High Sierra and later -# https://github.com/pypa/cibuildwheel/blob/ee63bf16da6cddfb925f542f2c7b59ad50e93969/action.yml#L31 -environment = { MACOSX_DEPLOYMENT_TARGET=10.13, DISABLE_NUMCODECS_AVX2=1, CFLAGS="$CFLAGS -Wno-implicit-function-declaration" } +# AVX2 disabled for portable wheels (not all x86_64 CPUs have it) +config-settings = "setup-args=-Davx2=disabled" + [[tool.cibuildwheel.overrides]] select = "*-macosx_arm64" -environment = { DISABLE_NUMCODECS_AVX2=1, DISABLE_NUMCODECS_SSE2=1 } +config-settings = {setup-args = ["-Davx2=disabled", "-Dsse2=disabled"]} [tool.ruff] line-length = 100 extend-exclude = ["c-blosc"] -src = ["numcodecs"] [tool.ruff.lint] extend-select = [ @@ -246,6 +240,12 @@ warn_unused_ignores = true warn_unused_configs = true [tool.uv] +# meson-python editable installs require build deps (ninja, meson) at runtime for +# auto-rebuild on import. Skip build isolation so the venv's ninja is used. +# The "dev" dependency group provides these build tools. Bootstrap with: +# `uv pip install --group dev` before running +# `uv pip install --no-build-isolation -e ".[test,test_extras,msgpack]"`. +no-build-isolation-package = ["numcodecs"] conflicts = [ # Zarr versions conflict with each other [ @@ -254,40 +254,3 @@ conflicts = [ { group = "test-zarr-main" } ] ] - -[tool.pixi.workspace] -channels = ["conda-forge"] -platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] - -[tool.pixi.dependencies] -python = "=3.14" -clang = ">=19.1.7,<20" -c-compiler = ">=1.9.0,<2" -cxx-compiler = ">=1.9.0,<2" -uv = "*" - -[tool.pixi.feature.test.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","zfpy"] } - -[tool.pixi.feature.test-google-crc32c.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] } - -[tool.pixi.feature.test-crc32c.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["crc32c"] } - -[tool.pixi.environments] -default = { solve-group = "default" } -test = ["test"] -test-crc32c = ["test", "test-crc32c"] -test-google-crc32c = ["test", "test-google-crc32c"] - -[tool.pixi.tasks] -ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze" -ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze" -ls-deps-main = "uv run --group test-zarr-main uv pip freeze" -test-zarr-312 = "uv run --group test-zarr-312 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" -test-zarr-313 = "uv run --group test-zarr-313 pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" -test-zarr-main = "uv run --group test-zarr-main pytest --cov=numcodecs --cov-report=xml --cov-report=term tests/test_zarr3.py tests/test_zarr3_import.py" - -[tool.pixi.feature.test.tasks] -run-tests = "pytest -v" diff --git a/setup.py b/setup.py deleted file mode 100644 index c27fad359..000000000 --- a/setup.py +++ /dev/null @@ -1,385 +0,0 @@ -import os -import sys -from distutils import ccompiler -from distutils.command.clean import clean -from distutils.sysconfig import customize_compiler -from glob import glob - -import cpuinfo -from Cython.Distutils.build_ext import new_build_ext as build_ext -from setuptools import Extension, setup -from setuptools.errors import CCompilerError, ExecError, PlatformError - -# determine CPU support for SSE2 and AVX2 -cpu_info = cpuinfo.get_cpu_info() -flags = cpu_info.get('flags', []) -machine = cpuinfo.platform.machine() - -# only check for x86 features on x86_64 arch -have_sse2 = False -have_avx2 = False -if machine == 'x86_64': - have_sse2 = 'sse2' in flags - have_avx2 = 'avx2' in flags - -disable_sse2 = 'DISABLE_NUMCODECS_SSE2' in os.environ -disable_avx2 = 'DISABLE_NUMCODECS_AVX2' in os.environ - -# setup common compile arguments -have_cflags = 'CFLAGS' in os.environ -base_compile_args = [] -if have_cflags: - # respect compiler options set by user - pass -elif os.name == 'posix' and machine == 'x86_64': - if disable_sse2: - base_compile_args.append('-mno-sse2') - elif have_sse2: - base_compile_args.append('-msse2') - if disable_avx2: - base_compile_args.append('-mno-avx2') - elif have_avx2: - base_compile_args.append('-mavx2') -# On macOS, force libc++ in case the system tries to use `stdlibc++`. -# The latter is often absent from modern macOS systems. -if sys.platform == 'darwin': - base_compile_args.append('-stdlib=libc++') - - -def info(*msg): - kwargs = {'file': sys.stdout} - print('[numcodecs]', *msg, **kwargs) - - -def error(*msg): - kwargs = {'file': sys.stderr} - print('[numcodecs]', *msg, **kwargs) - - -def blosc_extension(): - info('setting up Blosc extension') - - extra_compile_args = base_compile_args.copy() - extra_link_args = [] - define_macros = [] - - # ensure pthread is properly linked on POSIX systems - if os.name == 'posix': - extra_compile_args.append('-pthread') - extra_link_args.append('-pthread') - - # setup blosc sources - blosc_sources = [f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f] - include_dirs = [os.path.join('c-blosc', 'blosc')] - - # add internal complibs - blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c') - blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc') - blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') - blosc_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*/*') if os.path.isdir(d)] - # remove minizip because Python.h 3.8 tries to include crypt.h - include_dirs = [d for d in include_dirs if 'minizip' not in d] - define_macros += [ - ('HAVE_LZ4', 1), - # ('HAVE_SNAPPY', 1), - ('HAVE_ZLIB', 1), - ('HAVE_ZSTD', 1), - ] - # define_macros += [('CYTHON_TRACE', '1')] - - # SSE2 - if have_sse2 and not disable_sse2: - info('compiling Blosc extension with SSE2 support') - extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED') - blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f] - if os.name == 'nt': - define_macros += [('__SSE2__', 1)] - else: - info('compiling Blosc extension without SSE2 support') - - # AVX2 - if have_avx2 and not disable_avx2: - info('compiling Blosc extension with AVX2 support') - extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') - blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] - if os.name == 'nt': - define_macros += [('__AVX2__', 1)] - else: - info('compiling Blosc extension without AVX2 support') - - # include assembly files - if cpuinfo.platform.machine() == 'x86_64': - extra_objects = [ - S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") - ] - else: - extra_objects = [] - - sources = ['numcodecs/blosc.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.blosc', - sources=sources + blosc_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - extra_objects=extra_objects, - ), - ] - - -def zstd_extension(): - info('setting up Zstandard extension') - - zstd_sources = [] - extra_compile_args = base_compile_args.copy() - include_dirs = [] - define_macros = [] - - # setup sources - use zstd bundled in blosc - zstd_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') - zstd_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*') if os.path.isdir(d)] - include_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*/*') if os.path.isdir(d)] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/zstd.pyx'] - - # include assembly files - if cpuinfo.platform.machine() == 'x86_64': - extra_objects = [ - S[:-1] + 'o' for S in glob("c-blosc/internal-complibs/zstd*/decompress/*amd64.S") - ] - else: - extra_objects = [] - - # define extension module - return [ - Extension( - 'numcodecs.zstd', - sources=sources + zstd_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_objects=extra_objects, - ), - ] - - -def lz4_extension(): - info('setting up LZ4 extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - use LZ4 bundled in blosc - lz4_sources = glob('c-blosc/internal-complibs/lz4*/*.c') - include_dirs = [d for d in glob('c-blosc/internal-complibs/lz4*') if os.path.isdir(d)] - include_dirs += ['numcodecs'] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/lz4.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.lz4', - sources=sources + lz4_sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def vlen_extension(): - info('setting up vlen extension') - import numpy - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs', numpy.get_include()] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/vlen.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.vlen', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def fletcher_extension(): - info('setting up fletcher32 extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs'] - # define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/fletcher32.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.fletcher32', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def jenkins_extension(): - info('setting up jenkins extension') - - extra_compile_args = base_compile_args.copy() - define_macros = [] - - # setup sources - include_dirs = ['numcodecs'] - define_macros += [('CYTHON_TRACE', '1')] - - sources = ['numcodecs/jenkins.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.jenkins', - sources=sources, - include_dirs=include_dirs, - define_macros=define_macros, - extra_compile_args=extra_compile_args, - ), - ] - - -def compat_extension(): - info('setting up compat extension') - - extra_compile_args = base_compile_args.copy() - - sources = ['numcodecs/compat_ext.pyx'] - - # define extension module - return [ - Extension( - 'numcodecs.compat_ext', - sources=sources, - extra_compile_args=extra_compile_args, - ), - ] - - -def shuffle_extension(): - info('setting up shuffle extension') - - extra_compile_args = base_compile_args.copy() - - sources = ['numcodecs/_shuffle.pyx'] - - # define extension module - return [ - Extension('numcodecs._shuffle', sources=sources, extra_compile_args=extra_compile_args), - ] - - -if sys.platform == 'win32': - ext_errors = (CCompilerError, ExecError, PlatformError, IOError, ValueError) -else: - ext_errors = (CCompilerError, ExecError, PlatformError) - - -class BuildFailed(Exception): - pass - - -class ve_build_ext(build_ext): - # This class allows C extension building to fail. - - def run(self): - try: - machine = cpuinfo.platform.machine() - if machine in ('x86_64', 'aarch64'): - pattern = '*amd64.S' if machine == 'x86_64' else '*aarch64.S' - S_files = glob(f'c-blosc/internal-complibs/zstd*/decompress/{pattern}') - compiler = ccompiler.new_compiler() - customize_compiler(compiler) - compiler.src_extensions.append('.S') - compiler.compile(S_files) - - build_ext.run(self) - except PlatformError as e: - error(e) - raise BuildFailed from e - - def build_extension(self, ext): - try: - build_ext.build_extension(self, ext) - except ext_errors as e: - error(e) - raise BuildFailed from e - - -class Sclean(clean): - # Clean up .o files created by .S files - - def run(self): - if cpuinfo.platform.machine() == 'x86_64': - o_files = glob('c-blosc/internal-complibs/zstd*/decompress/*amd64.o') - for f in o_files: - os.remove(f) - - clean.run(self) - - -def run_setup(with_extensions): - if with_extensions: - ext_modules = ( - blosc_extension() - + zstd_extension() - + lz4_extension() - + compat_extension() - + shuffle_extension() - + vlen_extension() - + fletcher_extension() - + jenkins_extension() - ) - - cmdclass = {'build_ext': ve_build_ext, 'clean': Sclean} - else: - ext_modules = [] - cmdclass = {} - - setup( - ext_modules=ext_modules, - cmdclass=cmdclass, - ) - - -if __name__ == '__main__': - is_pypy = hasattr(sys, 'pypy_translation_info') - with_extensions = not is_pypy and 'DISABLE_NUMCODECS_CEXT' not in os.environ - run_setup(with_extensions) diff --git a/numcodecs/__init__.py b/src/numcodecs/__init__.py similarity index 97% rename from numcodecs/__init__.py rename to src/numcodecs/__init__.py index e3626b1b6..990341618 100644 --- a/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -20,10 +20,12 @@ import atexit import multiprocessing from contextlib import suppress +from importlib.metadata import version as _version from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec -from numcodecs.version import version as __version__ # noqa: F401 + +__version__: str = _version("numcodecs") from numcodecs.zlib import Zlib register_codec(Zlib) diff --git a/src/numcodecs/_build_utils/gitversion.py b/src/numcodecs/_build_utils/gitversion.py new file mode 100755 index 000000000..32c5165dc --- /dev/null +++ b/src/numcodecs/_build_utils/gitversion.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Get the version string for numcodecs, used by meson.build at configure time.""" + +import os + + +def get_version(): + try: + from setuptools_scm import get_version + + return get_version(root=os.path.join(os.path.dirname(__file__), '..', '..', '..')) + except (ImportError, LookupError): + pass + + # Fallback: read from pyproject.toml + pyproject = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'pyproject.toml') + with open(pyproject) as f: + for line in f: + if line.strip().startswith('version'): + return line.split('=')[1].strip().strip('"').strip("'") + + return '0.0.0' + + +if __name__ == '__main__': + print(get_version()) diff --git a/numcodecs/_shuffle.pyx b/src/numcodecs/_shuffle.pyx similarity index 100% rename from numcodecs/_shuffle.pyx rename to src/numcodecs/_shuffle.pyx diff --git a/numcodecs/_utils.pxd b/src/numcodecs/_utils.pxd similarity index 100% rename from numcodecs/_utils.pxd rename to src/numcodecs/_utils.pxd diff --git a/numcodecs/abc.py b/src/numcodecs/abc.py similarity index 100% rename from numcodecs/abc.py rename to src/numcodecs/abc.py diff --git a/numcodecs/astype.py b/src/numcodecs/astype.py similarity index 100% rename from numcodecs/astype.py rename to src/numcodecs/astype.py diff --git a/numcodecs/base64.py b/src/numcodecs/base64.py similarity index 100% rename from numcodecs/base64.py rename to src/numcodecs/base64.py diff --git a/numcodecs/bitround.py b/src/numcodecs/bitround.py similarity index 100% rename from numcodecs/bitround.py rename to src/numcodecs/bitround.py diff --git a/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx similarity index 100% rename from numcodecs/blosc.pyx rename to src/numcodecs/blosc.pyx diff --git a/numcodecs/bz2.py b/src/numcodecs/bz2.py similarity index 100% rename from numcodecs/bz2.py rename to src/numcodecs/bz2.py diff --git a/numcodecs/categorize.py b/src/numcodecs/categorize.py similarity index 100% rename from numcodecs/categorize.py rename to src/numcodecs/categorize.py diff --git a/numcodecs/checksum32.py b/src/numcodecs/checksum32.py similarity index 100% rename from numcodecs/checksum32.py rename to src/numcodecs/checksum32.py diff --git a/numcodecs/compat.py b/src/numcodecs/compat.py similarity index 100% rename from numcodecs/compat.py rename to src/numcodecs/compat.py diff --git a/numcodecs/compat_ext.pxd b/src/numcodecs/compat_ext.pxd similarity index 100% rename from numcodecs/compat_ext.pxd rename to src/numcodecs/compat_ext.pxd diff --git a/numcodecs/compat_ext.pyx b/src/numcodecs/compat_ext.pyx similarity index 100% rename from numcodecs/compat_ext.pyx rename to src/numcodecs/compat_ext.pyx diff --git a/numcodecs/delta.py b/src/numcodecs/delta.py similarity index 100% rename from numcodecs/delta.py rename to src/numcodecs/delta.py diff --git a/numcodecs/errors.py b/src/numcodecs/errors.py similarity index 100% rename from numcodecs/errors.py rename to src/numcodecs/errors.py diff --git a/numcodecs/fixedscaleoffset.py b/src/numcodecs/fixedscaleoffset.py similarity index 100% rename from numcodecs/fixedscaleoffset.py rename to src/numcodecs/fixedscaleoffset.py diff --git a/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx similarity index 100% rename from numcodecs/fletcher32.pyx rename to src/numcodecs/fletcher32.pyx diff --git a/numcodecs/gzip.py b/src/numcodecs/gzip.py similarity index 100% rename from numcodecs/gzip.py rename to src/numcodecs/gzip.py diff --git a/numcodecs/jenkins.pyx b/src/numcodecs/jenkins.pyx similarity index 100% rename from numcodecs/jenkins.pyx rename to src/numcodecs/jenkins.pyx diff --git a/numcodecs/json.py b/src/numcodecs/json.py similarity index 100% rename from numcodecs/json.py rename to src/numcodecs/json.py diff --git a/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx similarity index 100% rename from numcodecs/lz4.pyx rename to src/numcodecs/lz4.pyx diff --git a/numcodecs/lzma.py b/src/numcodecs/lzma.py similarity index 100% rename from numcodecs/lzma.py rename to src/numcodecs/lzma.py diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build new file mode 100644 index 000000000..eca2b2c64 --- /dev/null +++ b/src/numcodecs/meson.build @@ -0,0 +1,286 @@ +# --- Compiler flags --- +c_args = [] +link_args = [] + +if host_machine.system() != 'windows' + c_args += ['-pthread'] + link_args += ['-pthread'] +endif + +# --- SIMD detection --- +# Detected via compiler capability, NOT runtime CPU flags. +# Correct for cross-compilation: checks what the *target* supports. +have_sse2 = false +have_avx2 = false + +if host_machine.cpu_family() == 'x86_64' + # Honor legacy environment variables for backwards compatibility with + # existing build scripts and packaging recipes. + disable_sse2_env = run_command( + py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_SSE2", ""))', + check: true, + ).stdout().strip() != '' + disable_avx2_env = run_command( + py, '-c', 'import os; print(os.environ.get("DISABLE_NUMCODECS_AVX2", ""))', + check: true, + ).stdout().strip() != '' + + if get_option('sse2').disabled() or disable_sse2_env + have_sse2 = false + else + have_sse2 = cc.has_argument('-msse2') + endif + + if get_option('avx2').disabled() or disable_avx2_env + have_avx2 = false + else + have_avx2 = cc.has_argument('-mavx2') + endif +endif + +# --- Vendored zstd --- +zstd_dep = dependency('libzstd', required: get_option('system_zstd')) + +if not zstd_dep.found() + zstd_sources = files( + # common/ + '../../c-blosc/internal-complibs/zstd-1.5.6/common/debug.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/entropy_common.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/error_private.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/fse_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/pool.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/threading.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/xxhash.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/common/zstd_common.c', + # compress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/fse_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/hist.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/huf_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_literals.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_sequences.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_compress_superblock.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_double_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_fast.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_lazy.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_ldm.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstd_opt.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress/zstdmt_compress.c', + # decompress/ + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_ddict.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/zstd_decompress_block.c', + # dictBuilder/ + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/cover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/divsufsort.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/fastcover.c', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder/zdict.c', + ) + + # Assembly files — meson handles .S natively + if host_machine.cpu_family() == 'x86_64' + zstd_sources += files( + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress/huf_decompress_amd64.S', + ) + endif + + zstd_inc = include_directories( + '../../c-blosc/internal-complibs/zstd-1.5.6', + '../../c-blosc/internal-complibs/zstd-1.5.6/common', + '../../c-blosc/internal-complibs/zstd-1.5.6/compress', + '../../c-blosc/internal-complibs/zstd-1.5.6/decompress', + '../../c-blosc/internal-complibs/zstd-1.5.6/dictBuilder', + ) + + zstd_lib = static_library('zstd_vendored', zstd_sources, + include_directories: zstd_inc, + c_args: c_args, + ) + zstd_dep = declare_dependency( + link_with: zstd_lib, + include_directories: zstd_inc, + ) +endif + +# --- Vendored lz4 --- +lz4_dep = dependency('liblz4', required: get_option('system_lz4')) + +if not lz4_dep.found() + lz4_sources = files( + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4.c', + '../../c-blosc/internal-complibs/lz4-1.10.0/lz4hc.c', + ) + lz4_inc = include_directories('../../c-blosc/internal-complibs/lz4-1.10.0') + + lz4_lib = static_library('lz4_vendored', lz4_sources, + include_directories: lz4_inc, + c_args: c_args, + ) + lz4_dep = declare_dependency( + link_with: lz4_lib, + include_directories: lz4_inc, + ) +endif + +# --- Vendored zlib (always vendored, used by blosc) --- +zlib_sources = files( + '../../c-blosc/internal-complibs/zlib-1.3.1/adler32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/compress.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/crc32.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/deflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzclose.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzlib.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzread.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/gzwrite.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/infback.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inffast.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inflate.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/inftrees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/trees.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/uncompr.c', + '../../c-blosc/internal-complibs/zlib-1.3.1/zutil.c', +) +zlib_inc = include_directories('../../c-blosc/internal-complibs/zlib-1.3.1') + +zlib_lib = static_library('zlib_vendored', zlib_sources, + include_directories: zlib_inc, + c_args: c_args, +) +zlib_dep = declare_dependency( + link_with: zlib_lib, + include_directories: zlib_inc, +) + +# --- Vendored blosc --- +blosc_dep = dependency('blosc', required: get_option('system_blosc')) + +if not blosc_dep.found() + blosc_sources = files( + '../../c-blosc/blosc/blosc.c', + '../../c-blosc/blosc/blosclz.c', + '../../c-blosc/blosc/fastcopy.c', + '../../c-blosc/blosc/shuffle.c', + '../../c-blosc/blosc/shuffle-generic.c', + '../../c-blosc/blosc/bitshuffle-generic.c', + ) + + blosc_c_args = c_args + ['-DHAVE_LZ4', '-DHAVE_ZLIB', '-DHAVE_ZSTD'] + + if have_sse2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-sse2.c', + '../../c-blosc/blosc/bitshuffle-sse2.c', + ) + blosc_c_args += ['-DSHUFFLE_SSE2_ENABLED', '-msse2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__SSE2__'] + endif + endif + + if have_avx2 + blosc_sources += files( + '../../c-blosc/blosc/shuffle-avx2.c', + '../../c-blosc/blosc/bitshuffle-avx2.c', + ) + blosc_c_args += ['-DSHUFFLE_AVX2_ENABLED', '-mavx2'] + if host_machine.system() == 'windows' + blosc_c_args += ['-D__AVX2__'] + endif + endif + + blosc_inc = include_directories('../../c-blosc/blosc') + + blosc_lib = static_library('blosc_vendored', blosc_sources, + include_directories: blosc_inc, + dependencies: [lz4_dep, zlib_dep, zstd_dep], + c_args: blosc_c_args, + ) + blosc_dep = declare_dependency( + link_with: blosc_lib, + include_directories: blosc_inc, + ) +endif + +# --- Cython extension modules --- + +# Extensions with C library dependencies +py.extension_module('blosc', + 'blosc.pyx', + dependencies: [blosc_dep, lz4_dep, zlib_dep, zstd_dep], + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('zstd', + 'zstd.pyx', + dependencies: [zstd_dep], + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('lz4', + 'lz4.pyx', + dependencies: [lz4_dep], + include_directories: include_directories('.'), + c_args: c_args, + link_args: link_args, + install: true, + subdir: 'numcodecs', +) + +py.extension_module('vlen', + 'vlen.pyx', + dependencies: [numpy_dep], + include_directories: include_directories('.'), + c_args: c_args, + install: true, + subdir: 'numcodecs', +) + +# Pure-Cython extensions (no C library deps) +foreach ext : ['fletcher32', 'jenkins', 'compat_ext', '_shuffle'] + py.extension_module(ext, + ext + '.pyx', + include_directories: include_directories('.'), + c_args: c_args, + install: true, + subdir: 'numcodecs', + ) +endforeach + +# --- Pure Python sources --- +py.install_sources( + '__init__.py', + 'abc.py', + 'astype.py', + 'base64.py', + 'bitround.py', + 'bz2.py', + 'categorize.py', + 'checksum32.py', + 'compat.py', + 'delta.py', + 'errors.py', + 'fixedscaleoffset.py', + 'gzip.py', + 'json.py', + 'lzma.py', + 'msgpacks.py', + 'ndarray_like.py', + 'packbits.py', + 'pcodec.py', + 'pickles.py', + 'quantize.py', + 'registry.py', + 'shuffle.py', + 'version.py', + 'zarr3.py', + 'zfpy.py', + 'zlib.py', + subdir: 'numcodecs', +) diff --git a/numcodecs/msgpacks.py b/src/numcodecs/msgpacks.py similarity index 100% rename from numcodecs/msgpacks.py rename to src/numcodecs/msgpacks.py diff --git a/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py similarity index 100% rename from numcodecs/ndarray_like.py rename to src/numcodecs/ndarray_like.py diff --git a/numcodecs/packbits.py b/src/numcodecs/packbits.py similarity index 100% rename from numcodecs/packbits.py rename to src/numcodecs/packbits.py diff --git a/numcodecs/pcodec.py b/src/numcodecs/pcodec.py similarity index 99% rename from numcodecs/pcodec.py rename to src/numcodecs/pcodec.py index e214f7812..79732f2fd 100644 --- a/numcodecs/pcodec.py +++ b/src/numcodecs/pcodec.py @@ -1,8 +1,9 @@ from typing import Literal +from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone + from numcodecs.abc import Codec from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray -from pcodec import ChunkConfig, DeltaSpec, ModeSpec, PagingSpec, standalone DEFAULT_MAX_PAGE_N = 262144 diff --git a/numcodecs/pickles.py b/src/numcodecs/pickles.py similarity index 100% rename from numcodecs/pickles.py rename to src/numcodecs/pickles.py diff --git a/numcodecs/quantize.py b/src/numcodecs/quantize.py similarity index 100% rename from numcodecs/quantize.py rename to src/numcodecs/quantize.py diff --git a/numcodecs/registry.py b/src/numcodecs/registry.py similarity index 100% rename from numcodecs/registry.py rename to src/numcodecs/registry.py diff --git a/numcodecs/shuffle.py b/src/numcodecs/shuffle.py similarity index 100% rename from numcodecs/shuffle.py rename to src/numcodecs/shuffle.py diff --git a/src/numcodecs/version.py b/src/numcodecs/version.py new file mode 100644 index 000000000..b69bb5a28 --- /dev/null +++ b/src/numcodecs/version.py @@ -0,0 +1,7 @@ +# Backwards-compatibility shim. The version is now provided by importlib.metadata +# rather than being generated by setuptools-scm. This module exists so that +# `from numcodecs.version import version` continues to work. +from importlib.metadata import version as _version + +__version__: str = _version("numcodecs") +version: str = __version__ diff --git a/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx similarity index 100% rename from numcodecs/vlen.pyx rename to src/numcodecs/vlen.pyx diff --git a/numcodecs/zarr3.py b/src/numcodecs/zarr3.py similarity index 100% rename from numcodecs/zarr3.py rename to src/numcodecs/zarr3.py diff --git a/numcodecs/zfpy.py b/src/numcodecs/zfpy.py similarity index 100% rename from numcodecs/zfpy.py rename to src/numcodecs/zfpy.py diff --git a/numcodecs/zlib.py b/src/numcodecs/zlib.py similarity index 100% rename from numcodecs/zlib.py rename to src/numcodecs/zlib.py diff --git a/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx similarity index 100% rename from numcodecs/zstd.pyx rename to src/numcodecs/zstd.pyx diff --git a/tests/common.py b/tests/common.py index 31685d955..e3c9cfb3e 100644 --- a/tests/common.py +++ b/tests/common.py @@ -6,10 +6,11 @@ import numpy as np import pytest +from numpy.testing import assert_array_almost_equal, assert_array_equal + from numcodecs import * # noqa: F403 # for eval to find names in repr tests from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.registry import get_codec -from numpy.testing import assert_array_almost_equal, assert_array_equal greetings = [ '¡Hola mundo!', diff --git a/tests/test_astype.py b/tests/test_astype.py index d0cc4c596..4022e2f5e 100644 --- a/tests/test_astype.py +++ b/tests/test_astype.py @@ -1,7 +1,7 @@ import numpy as np -from numcodecs.astype import AsType from numpy.testing import assert_array_equal +from numcodecs.astype import AsType from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_base64.py b/tests/test_base64.py index 71d671c7b..39d118034 100644 --- a/tests/test_base64.py +++ b/tests/test_base64.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from numcodecs.base64 import Base64 +from numcodecs.base64 import Base64 from tests.common import ( check_backwards_compatibility, check_encode_decode, diff --git a/tests/test_bitround.py b/tests/test_bitround.py index 663cf73a3..3929e43c6 100644 --- a/tests/test_bitround.py +++ b/tests/test_bitround.py @@ -1,5 +1,6 @@ import numpy as np import pytest + from numcodecs.bitround import BitRound, max_bits # adapted from https://github.com/milankl/BitInformation.jl/blob/main/test/round_nearest.jl diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 6e1927302..537dc3505 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -5,8 +5,9 @@ import pytest try: - from numcodecs import blosc from numcodecs.blosc import Blosc + + from numcodecs import blosc except ImportError: # pragma: no cover pytest.skip("numcodecs.blosc not available", allow_module_level=True) diff --git a/tests/test_bz2.py b/tests/test_bz2.py index a58446ca2..74ed5534e 100644 --- a/tests/test_bz2.py +++ b/tests/test_bz2.py @@ -1,8 +1,8 @@ import itertools import numpy as np -from numcodecs.bz2 import BZ2 +from numcodecs.bz2 import BZ2 from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_categorize.py b/tests/test_categorize.py index 8431d4fb9..97aca9770 100644 --- a/tests/test_categorize.py +++ b/tests/test_categorize.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from numcodecs.categorize import Categorize from numpy.testing import assert_array_equal +from numcodecs.categorize import Categorize from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_checksum32.py b/tests/test_checksum32.py index fe51084b3..0a6db0c7d 100644 --- a/tests/test_checksum32.py +++ b/tests/test_checksum32.py @@ -4,8 +4,8 @@ import numpy as np import pytest -from numcodecs.checksum32 import CRC32, Adler32 +from numcodecs.checksum32 import CRC32, Adler32 from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_compat.py b/tests/test_compat.py index 091e34539..0d5f2d74e 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -3,6 +3,7 @@ import numpy as np import pytest + from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_text diff --git a/tests/test_delta.py b/tests/test_delta.py index 08df959b9..df48d6d06 100644 --- a/tests/test_delta.py +++ b/tests/test_delta.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from numcodecs.delta import Delta from numpy.testing import assert_array_equal +from numcodecs.delta import Delta from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_entrypoints.py b/tests/test_entrypoints.py index 25eb9d34d..43503ec39 100644 --- a/tests/test_entrypoints.py +++ b/tests/test_entrypoints.py @@ -1,9 +1,10 @@ import os.path import sys -import numcodecs.registry import pytest +import numcodecs.registry + here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_entrypoints_backport.py b/tests/test_entrypoints_backport.py index ff1b02a67..7e1c32bcc 100644 --- a/tests/test_entrypoints_backport.py +++ b/tests/test_entrypoints_backport.py @@ -3,9 +3,10 @@ import sys from multiprocessing import Process -import numcodecs.registry import pytest +import numcodecs.registry + importlib_spec = importlib.util.find_spec("importlib_metadata") if importlib_spec is None or importlib_spec.loader is None: # pragma: no cover pytest.skip( diff --git a/tests/test_fixedscaleoffset.py b/tests/test_fixedscaleoffset.py index 9c9e36dfb..343d30263 100644 --- a/tests/test_fixedscaleoffset.py +++ b/tests/test_fixedscaleoffset.py @@ -2,9 +2,9 @@ import numpy as np import pytest -from numcodecs.fixedscaleoffset import FixedScaleOffset from numpy.testing import assert_array_equal +from numcodecs.fixedscaleoffset import FixedScaleOffset from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_gzip.py b/tests/test_gzip.py index 745b6b025..5031231e7 100644 --- a/tests/test_gzip.py +++ b/tests/test_gzip.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from numcodecs.gzip import GZip +from numcodecs.gzip import GZip from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 2187934b1..40d972bf7 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -1,8 +1,9 @@ import numpy as np import pytest -from numcodecs.checksum32 import JenkinsLookup3 from numcodecs.jenkins import jenkins_lookup3 +from numcodecs.checksum32 import JenkinsLookup3 + def test_jenkins_lookup3(): h = jenkins_lookup3(b"", 0) diff --git a/tests/test_json.py b/tests/test_json.py index 94a8b1c2a..c8b2cbca0 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from numcodecs.json import JSON +from numcodecs.json import JSON from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_ndarray_like.py b/tests/test_ndarray_like.py index ff10e2400..6c16e7dbf 100644 --- a/tests/test_ndarray_like.py +++ b/tests/test_ndarray_like.py @@ -1,4 +1,5 @@ import pytest + from numcodecs.ndarray_like import DType, FlagsObj, NDArrayLike diff --git a/tests/test_packbits.py b/tests/test_packbits.py index 10f058daf..8bda1cd33 100644 --- a/tests/test_packbits.py +++ b/tests/test_packbits.py @@ -1,6 +1,6 @@ import numpy as np -from numcodecs.packbits import PackBits +from numcodecs.packbits import PackBits from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_pickles.py b/tests/test_pickles.py index 3012f6110..fdabd459c 100644 --- a/tests/test_pickles.py +++ b/tests/test_pickles.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from numcodecs.pickles import Pickle +from numcodecs.pickles import Pickle from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_quantize.py b/tests/test_quantize.py index f29aae4db..724a7ba82 100644 --- a/tests/test_quantize.py +++ b/tests/test_quantize.py @@ -2,9 +2,9 @@ import numpy as np import pytest -from numcodecs.quantize import Quantize from numpy.testing import assert_array_almost_equal, assert_array_equal +from numcodecs.quantize import Quantize from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/test_registry.py b/tests/test_registry.py index a5e52ba01..93ed8f032 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,7 +1,8 @@ import inspect -import numcodecs import pytest + +import numcodecs from numcodecs.errors import UnknownCodecError from numcodecs.registry import get_codec diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 000000000..a1e30d9a1 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,15 @@ +import numcodecs + + +def test_version(): + assert isinstance(numcodecs.__version__, str) + assert numcodecs.__version__ + + +def test_version_module(): + from numcodecs.version import __version__, version + + assert isinstance(__version__, str) + assert isinstance(version, str) + assert __version__ == version + assert __version__ == numcodecs.__version__ diff --git a/tests/test_zlib.py b/tests/test_zlib.py index c5043722b..82e692b7f 100644 --- a/tests/test_zlib.py +++ b/tests/test_zlib.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from numcodecs.zlib import Zlib +from numcodecs.zlib import Zlib from tests.common import ( check_backwards_compatibility, check_config,