diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 5fa97d3b5e3..6c0d918d418 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -102,6 +102,7 @@ jobs: echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV" echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV" echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV" + echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV" if [ "$RUNNER_OS" == "Linux" ]; then sudo apt-get update -qq && sudo apt-get install -y ccache locales sudo locale-gen de_DE.UTF-8 @@ -118,6 +119,16 @@ jobs: restore-keys: | ccache-v4-${{ runner.os }}-${{ matrix.name }}- + - name: Restore CastXML cache + if: matrix.python-version != '' + id: restore-castxml-cache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + restore-keys: | + castxml-v1-${{ runner.os }}-arm-python- + - name: Restore ExternalData object store id: restore-externaldata uses: actions/cache/restore@v5 @@ -202,6 +213,13 @@ jobs: path: ${{ runner.temp }}/ccache key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }} + - name: Save CastXML cache + if: ${{ !cancelled() && matrix.python-version != '' }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }} + # ExternalData object store is populated by # .github/workflows/populate-externaldata-cache.yml — a dedicated # workflow whose only job is to prefetch every CID and write the diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000000..267704d0363 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,179 @@ +name: ITK.Pixi.Python + +on: + push: + branches: + - main + - 'release*' + paths-ignore: + - '*.md' + - LICENSE + - NOTICE + - 'Documentation/**' + - 'Utilities/Debugger/**' + - 'Utilities/ITKv5Preparation/**' + - 'Utilities/Maintenance/**' + - 'Modules/Remote/*.remote.cmake' + pull_request: + paths-ignore: + - '*.md' + - LICENSE + - NOTICE + - 'Documentation/**' + - 'Utilities/Debugger/**' + - 'Utilities/ITKv5Preparation/**' + - 'Utilities/Maintenance/**' + - 'Modules/Remote/*.remote.cmake' + +concurrency: + group: '${{ github.workflow }}@${{ github.head_ref || github.ref }}' + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + Pixi-Python: + runs-on: ${{ matrix.os }} + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + # windows-2022 excluded: itk_end_wrap_module.cmake generates an + # igenerator command that exceeds cmd.exe's 8191-char batch-file + # line limit for large modules (e.g. ITKImageIntensity, 59 + # submodules). Fix requires response-file support in cmake custom + # command, tracked separately. + os: [ubuntu-24.04, macos-15] + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 5 + clean: true + + - name: Configure ccache environment + shell: bash + run: | + echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "$GITHUB_ENV" + echo "CCACHE_COMPILERCHECK=content" >> "$GITHUB_ENV" + echo "CCACHE_NOHASHDIR=true" >> "$GITHUB_ENV" + echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV" + echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV" + echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV" + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get update -qq && sudo apt-get install -y locales + sudo locale-gen de_DE.UTF-8 + fi + + - name: Configure CastXML cache environment + shell: bash + run: | + echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV" + + - name: Restore compiler cache + id: restore-ccache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/ccache + key: ccache-v4-${{ runner.os }}-pixi-python-${{ github.sha }} + restore-keys: | + ccache-v4-${{ runner.os }}-pixi-python- + + - name: Restore CastXML cache + id: restore-castxml-cache + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-pixi-python-${{ github.sha }} + restore-keys: | + castxml-v1-${{ runner.os }}-pixi-python- + + - name: Restore ExternalData object store + id: restore-externaldata + uses: actions/cache/restore@v5 + with: + path: ${{ runner.temp }}/ExternalData + key: externaldata-v1-${{ hashFiles('**/*.cid') }} + restore-keys: | + externaldata-v1- + + - name: Disk space before build (Ubuntu) + if: matrix.os == 'ubuntu-24.04' + run: df -h / + + - name: Free Disk Space (Ubuntu) + if: matrix.os == 'ubuntu-24.04' + uses: BRAINSia/free-disk-space@v2 + with: + removalmode: "rmz" + swap-storage: "true" + haskell: "true" + dotnet: "true" + docker-images: "false" + tool-cache: "true" + android: "false" + large-packages: "true" + mandb: "true" + + - name: Export ExternalData_OBJECT_STORES + shell: bash + run: | + echo "ExternalData_OBJECT_STORES=${{ runner.temp }}/ExternalData" >> "$GITHUB_ENV" + + - name: Set up Pixi + uses: prefix-dev/setup-pixi@v0.9.5 + + - name: Show ccache configuration, stats and maintenance + shell: bash + run: | + pixi run -e python ccache --zero-stats + pixi run -e python ccache --evict-older-than 7d + pixi run -e python ccache --show-config + + - name: Configure + run: pixi run configure-python-ci + + - name: Fetch ExternalData + shell: bash + run: pixi run -e python cmake --build build-python --target ITKData + + - name: Build + run: | + df -h / || true + pixi run build-python-ci + df -h / || true + + - name: Free disk space after build + shell: bash + run: | + find build-python -type f \( -path '*/CMakeFiles/*' -o -path '*.dir/*' \) \( -name "*.o" -o -name "*.obj" \) -delete + find build-python/lib -type f \( -name "*.a" -o -name "*.lib" \) -delete 2>/dev/null || true + pixi run -e python ccache --evict-older-than 1d 2>/dev/null || true + pixi run -e python ccache --cleanup 2>/dev/null || true + df -h / || true + + - name: Test + run: pixi run test-python-ci + + - name: Save compiler cache + if: ${{ !cancelled() }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/ccache + key: ccache-v4-${{ runner.os }}-pixi-python-${{ github.sha }} + + - name: Save CastXML cache + if: ${{ !cancelled() }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/itk-castxml-cache + key: castxml-v1-${{ runner.os }}-pixi-python-${{ github.sha }} + + - name: Save ExternalData object store + if: ${{ !cancelled() }} + uses: actions/cache/save@v5 + with: + path: ${{ runner.temp }}/ExternalData + key: externaldata-v1-${{ hashFiles('**/*.cid') }} + + - name: ccache stats + if: always() + run: pixi run -e python ccache --show-stats diff --git a/CMake/itkWrapCastXMLCacheSupport.cmake b/CMake/itkWrapCastXMLCacheSupport.cmake new file mode 100644 index 00000000000..f425e99cbe4 --- /dev/null +++ b/CMake/itkWrapCastXMLCacheSupport.cmake @@ -0,0 +1,60 @@ +############################################################################### +# Content-addressed two-level cache for CastXML wrapping steps. +# +# When ITK_WRAP_CASTXML_CACHE is ON, a Python wrapper replaces +# `ccache castxml` for every .xml generation step. The wrapper computes +# a two-level SHA-256 key: +# L1 (fast, ~0.2s): direct inputs (cxx + castxml.inc + compiler flags) +# L2 (robust, ~1s): sha256(L1_key + `castxml -E` preprocessed output) +# +# On L2 hit: restores .xml from cache, saving the full CastXML run. +# On miss: runs castxml normally and populates the cache. +# +# Cache location: $ITK_WRAP_CACHE env var (default: ~/.cache/itk-wrap) + +set( + _ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT + "${ITK_SOURCE_DIR}/Wrapping/Generators/CastXML/itk-castxml-cache.py" +) + +option( + ITK_WRAP_CASTXML_CACHE + "Use a content-addressed two-level cache for CastXML wrapping steps." + ON +) +mark_as_advanced(ITK_WRAP_CASTXML_CACHE) + +if(ITK_WRAP_CASTXML_CACHE) + set( + ITK_WRAP_CASTXML_CACHE_SCRIPT + "${_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT}" + CACHE FILEPATH + "Path to the CastXML content-addressed cache wrapper script" + ) + mark_as_advanced(ITK_WRAP_CASTXML_CACHE_SCRIPT) + + if(NOT EXISTS "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE is ON but the wrapper script was not found:\n" + " ${ITK_WRAP_CASTXML_CACHE_SCRIPT}\n" + "Set ITK_WRAP_CASTXML_CACHE_SCRIPT to the correct path or turn off ITK_WRAP_CASTXML_CACHE." + ) + endif() + + if(NOT Python3_EXECUTABLE) + message( + FATAL_ERROR + "ITK_WRAP_CASTXML_CACHE requires Python3_EXECUTABLE to be set." + ) + endif() + + message(STATUS "CastXML content-addressed cache enabled") + message(STATUS " Script: ${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + message( + STATUS + " Cache root: set ITK_WRAP_CACHE env var at build time (default: ~/.cache/itk-wrap)" + ) +endif() + +unset(_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT) diff --git a/Documentation/docs/contributing/index.md b/Documentation/docs/contributing/index.md index 69651949f96..de86176ec99 100644 --- a/Documentation/docs/contributing/index.md +++ b/Documentation/docs/contributing/index.md @@ -461,6 +461,7 @@ CDash Dashboard dashboard.md updating_third_party.md python_packaging.md +wrapping_architecture.md ../README.md ``` diff --git a/Documentation/docs/contributing/wrapping_architecture.md b/Documentation/docs/contributing/wrapping_architecture.md new file mode 100644 index 00000000000..70f5f3423b7 --- /dev/null +++ b/Documentation/docs/contributing/wrapping_architecture.md @@ -0,0 +1,221 @@ +Python Wrapping Architecture +============================ + +ITK's Python wrapping pipeline converts C++ template declarations into +importable Python extension modules (`.abi3.so`) and type-stub files +(`.pyi`). The pipeline runs in two distinct phases: a **configure phase** +driven by CMake and a **build phase** driven by Ninja. + +## Configure phase: `.wrap` → `castxml_inputs/` + +Each ITK module that supports wrapping contains a `wrapping/` subdirectory +with one `.wrap` file per submodule. A `.wrap` file is a CMake script that +calls macros such as `itk_wrap_class()` and `itk_wrap_template()` to declare +which C++ template instantiations should be exposed to Python. + +CMake processes every `.wrap` file at configure time and writes three +files per submodule into `/Wrapping/castxml_inputs/`: + +| Generated file | How | Contents | +|---|---|---| +| `.cxx` | `configure_file` | `#include` directives + `_wrapping_` namespace with `using` aliases for every requested template instantiation | +| `.castxml.inc` | `file(GENERATE …)` | Compiler `-I` and `-D` flags needed to parse the `.cxx` file | +| `SwigInterface.h.in` | `configure_file` | `#include` list used by SWIG | + +CMake also registers one `add_custom_command` per submodule (for CastXML) +and one per ITK module (for `igenerator.py`). No compilation happens at +configure time; only the input files and build rules are written. + +## Build phase: CastXML → igenerator → SWIG → compile → link + +### Step 1 — CastXML (816 independent jobs) + +Each submodule produces exactly one XML file: + +``` +castxml_inputs/.cxx +castxml_inputs/.castxml.inc ──▶ itk-castxml-cache.py ──▶ castxml_inputs/.xml +Modules/.../include/.h (many) +``` + +`itk-castxml-cache.py` wraps the CastXML binary with a two-level +content-addressed cache (`~/.cache/itk-wrap` or `$ITK_WRAP_CACHE`): + +- **L1** — hash of the `.cxx` file content → L2 key +- **L2** — `castxml -E` (preprocessor only) output hash → cached `output.xml.gz` + +A cache hit avoids running CastXML entirely. All 816 CastXML jobs are +independent and run fully in parallel. No CastXML job reads or modifies +another submodule's `.xml` output. + +### Step 2 — `igenerator.py` (96 per-module jobs) + +Each ITK module (e.g. `ITKImageIntensity`) batches all of its submodules +into a single `igenerator.py` invocation: + +``` +castxml_inputs/itkAbsImageFilter.xml ──┐ +castxml_inputs/itkImage.xml ──┤ +castxml_inputs/itkOffset.xml ──┤ igenerator.py [ITKImageIntensity] +... (all N submodule XMLs) ──┘ --submodule-order "sub1;sub2;...;subN" + │ + ┌──────────────────────────┼─────────────────────────────┐ + │ per submodule (×N) │ │ + ▼ ▼ ▼ + Typedefs/.i itk-pkl/.index.txt itk-pkl-v1.db (SQLite) + Typedefs/.idx (lists DB keys; byproduct) (class metadata; WAL mode) + Typedefs/SwigInterface.h + + per module (×1): + itk-pkl/.stamp + itk/Configuration/_snake_case.py +``` + +`igenerator.py` uses `pygccxml` to parse each `.xml` file and emit SWIG +interface (`.i`) and index (`.idx`) files, class-metadata rows in a shared +SQLite database consumed later by `pyi_generator.py`, and a `.index.txt` +manifest listing the DB keys for each submodule. + +The SQLite database (`itk-pkl-v1.db`) is written to the `itk-pkl/` directory +inside the build tree by default. Set `ITK_WRAP_CACHE` to redirect it to a +shared location (e.g. a CI cache volume). + +**Ninja scheduling**: an `igenerator.py` job for module A starts as soon +as all of A's CastXML jobs are complete, even while CastXML is still +running for module B. There is no global barrier between the CastXML and +`igenerator.py` layers. + +### Step 3 — SWIG, compile, link (per submodule) + +``` +Typedefs/.i +Typedefs/SwigInterface.h ──▶ swig ──▶ Modules/.../Python.cpp + Generators/Python/itk/Python.py + +Modules/.../Python.cpp ──▶ ccache + g++ ──▶ .o ──▶ link ──▶ _Python.abi3.so +``` + +### Step 4 — `pyi_generator.py` (one global job) + +After **all** 816 `.index.txt` files exist, `pyi_generator.py` reads every +`.index.txt`, queries the SQLite database for each key, and writes the +type-stub files used by IDEs: + +``` +itk-pkl/.index.txt (×816) ──▶ pyi_generator.py ──▶ _proxies.pyi +itk-pkl-v1.db (SQLite) (queries DB via keys in .index.txt) __init__.pyi +``` + +## Key file reference + +| Path pattern | Written by | Read by | +|---|---|---| +| `Wrapping/castxml_inputs/.cxx` | CMake `configure_file` | CastXML | +| `Wrapping/castxml_inputs/.castxml.inc` | CMake `file(GENERATE)` | `itk-castxml-cache.py` | +| `Wrapping/castxml_inputs/.xml` | `itk-castxml-cache.py` / CastXML | `igenerator.py` | +| `Wrapping/Typedefs/.i` | `igenerator.py` | SWIG | +| `Wrapping/Typedefs/.idx` | `igenerator.py` | SWIG | +| `Wrapping/Generators/Python/itk-pkl/.index.txt` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/itk-pkl-v1.db` | `igenerator.py` | `pyi_generator.py` | +| `Wrapping/Generators/Python/itk-pkl/.stamp` | `igenerator.py` | ninja (tracks DB write completeness) | +| `Wrapping/Generators/Python/itk/_Python.abi3.so` | linker | Python `import itk` | +| `Wrapping/Generators/Python/itk/_proxies.pyi` | `pyi_generator.py` | IDEs | + +## Caches + +### CastXML cache (`itk-castxml-cache.py`) + +Controls via environment: + +| Variable | Default | Purpose | +|---|---|---| +| `ITK_WRAP_CACHE` | `~/.cache/itk-wrap` | Cache root for CastXML `.xml.gz` files | +| `ITK_WRAP_CACHE_VERBOSE` | unset | Set to `1` to log HIT/MISS per file | + +The CastXML cache is content-addressed and generator-version-stamped +(`_KEY_VERSION` in `itk-castxml-cache.py`). It is shared across build +directories; a fresh configure reuses XML from a previous build on the same +machine. + +### pkl SQLite database (`igenerator.py` / `pyi_generator.py`) + +| Variable | Default | Purpose | +|---|---|---| +| `ITK_WRAP_CACHE` | *(build tree `itk-pkl/`)* | Redirect pkl DB to a shared location | + +The pkl database (`itk-pkl-v1.db`) defaults to the build tree's `itk-pkl/` +directory and is a build artifact, not a user-level cache. Developers who +want to share it across build trees (or populate it from CI) must opt in by +setting `ITK_WRAP_CACHE` explicitly: + +```bash +export ITK_WRAP_CACHE=~/.cache/itk-wrap # personal shared cache +export ITK_WRAP_CACHE=$(dirname "$CCACHE_DIR") # CI: same root as ccache +``` + +Both the CastXML cache and the pkl database use the same `ITK_WRAP_CACHE` +root, so a single variable covers both. + +### ccache + +CastXML re-runs produce identical `.xml` files (the content is deterministic) +but are slow. The CastXML cache eliminates that cost. For the C++ compilation +steps (Step 3), ccache caches compiled `.o` files keyed on source content. +Both caches are independent and complement each other. + +### Build-phase timing on 2-core CI runners + +On a 2-core CI runner (cold caches throughout): + +| Phase | Approx. time | Notes | +|---|---|---| +| CastXML (816 jobs, 2 cores) | ~32 min | Eliminated on warm-cache runs | +| igenerator + SWIG + C++ compile | ~225 min | Reduced by ccache on subsequent runs | +| Tests | ~44 min | | + +CastXML is ~10 % of the cold-cache total. The C++ compilation phase +dominates; `ccache` is the primary lever there. + +## Ninja dependency graph summary + +``` +.wrap files (configure time, not in graph) + │ cmake configure_file / file(GENERATE) + ▼ +castxml_inputs/.cxx + .castxml.inc + .h headers + │ itk-castxml-cache.py [816 parallel jobs] + ▼ +castxml_inputs/.xml (write-once; never mutated after creation) + │ igenerator.py [96 per-module jobs; starts per-module, not globally gated] + ▼ +Typedefs/.i + .idx + SwigInterface.h +itk-pkl/.index.txt (DB keys) + itk-pkl-v1.db (SQLite) + .stamp + │ swig + ccache compile + link [parallel per submodule] + ▼ +_Python.abi3.so + │ pyi_generator.py [1 global job; needs all .index.txt] + ▼ +_proxies.pyi + __init__.pyi +``` + +## Troubleshooting + +**`No pkl keys were found in index files in itk-pkl`** +: The `.index.txt` manifests exist (so ninja considers `igenerator.py` + up-to-date) but the pkl database is absent or stale. Delete the stamp + files to force `igenerator.py` to re-run and repopulate the DB: + ```bash + find /Wrapping/Generators/Python/itk-pkl -name "*.index.txt" -o -name "*.stamp" | xargs rm -f + ninja -C + ``` + +**CastXML cache not being used** +: Set `ITK_WRAP_CACHE_VERBOSE=1` and rebuild one module to confirm HIT or + MISS log lines. Ensure `ITK_WRAP_CASTXML_CACHE=ON` is set in CMake. + A version bump to `_KEY_VERSION` in `itk-castxml-cache.py` forces a cold + cache for all entries. + +**Adding a new wrapped class** +: Add `itk_wrap_class()` / `itk_wrap_template()` calls to the relevant + `.wrap` file. Re-run CMake to regenerate the `.cxx` and `.castxml.inc` + files, then build normally. CMake will automatically include the new + submodule in the `--submodule-order` passed to `igenerator.py`. diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinux.yml b/Testing/ContinuousIntegration/AzurePipelinesLinux.yml index 2f21bbe2344..b54357c18e5 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinux.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinux.yml @@ -55,6 +55,7 @@ jobs: df -h / displayName: 'Free preinstalled software' + - checkout: self clean: true fetchDepth: 5 diff --git a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml index 8cc9be6c2dc..b4d162752f8 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Linux timeoutInMinutes: 0 @@ -55,6 +56,7 @@ jobs: df -h / displayName: 'Free preinstalled software' + - checkout: self clean: true fetchDepth: 5 @@ -102,6 +104,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "LinuxPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml index 898ba9743cf..ee84c981241 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: macOS timeoutInMinutes: 0 @@ -106,6 +107,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "macOSPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml index 3e834a7dfda..c021e87db2e 100644 --- a/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml +++ b/Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml @@ -36,6 +36,7 @@ variables: CCACHE_NOHASHDIR: 'true' CCACHE_SLOPPINESS: pch_defines,time_macros CCACHE_MAXSIZE: 8G + ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache jobs: - job: Windows timeoutInMinutes: 0 @@ -84,6 +85,14 @@ jobs: path: $(CCACHE_DIR) displayName: 'Restore ccache' + - task: Cache@2 + inputs: + key: '"castxml-v1" | "$(Agent.OS)" | "WindowsPython" | "$(Build.SourceVersion)"' + restoreKeys: | + "castxml-v1" | "$(Agent.OS)" | "WindowsPython" + path: $(ITK_WRAP_CACHE) + displayName: 'Restore CastXML cache' + - bash: | set -x ccache --zero-stats diff --git a/Wrapping/CMakeLists.txt b/Wrapping/CMakeLists.txt index 1f4d8c95cd3..c975d5387f9 100644 --- a/Wrapping/CMakeLists.txt +++ b/Wrapping/CMakeLists.txt @@ -152,6 +152,10 @@ endif() include(ConfigureWrapping.cmake) +if(ITK_WRAP_PYTHON) + include(itkWrapCastXMLCacheSupport) +endif() + ############################################################################### # Configure specific wrapper modules ############################################################################### diff --git a/Wrapping/Generators/CastXML/itk-castxml-cache.py b/Wrapping/Generators/CastXML/itk-castxml-cache.py new file mode 100755 index 00000000000..8e34e1d4c5a --- /dev/null +++ b/Wrapping/Generators/CastXML/itk-castxml-cache.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +""" +Two-level content-addressed cache wrapper for ITK's CastXML wrapping step. + +Invocation (replaces ccache + castxml in the cmake COMMAND): + python3 itk-castxml-cache.py /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --no-cache /path/to/castxml [castxml args...] + python3 itk-castxml-cache.py --evict [--max-size 2G] + +Cache key hierarchy: + L1 (fast, no subprocess): sha256 of castxml content-hash + inc file + cxx file + flags. + The castxml binary is fingerprinted by SHA-256 of its content (not mtime), so + `ninja -t clean` re-links the binary without changing the L1 key. + L2 (content-addressed): sha256 of normalised `castxml -E` preprocessed output. + Preprocessor line markers (# N "path") are stripped before hashing, + making L2 keys path-independent across build directories. + +Lookup: + L1 HIT → stored L2 key → L2 entry exists → restore → DONE (no subprocess) + L1 miss → run castxml -E → compute L2 key + L2 HIT → restore; refresh L1 map + L2 miss → run full castxml; store; write L1 map + +Storage formats (ITK_WRAP_CACHE_FORMAT): + gzip (default): output.xml.gz, ~8x smaller than raw XML. Decompressed on + restore. 253 MB for a full 807-module ITK build; each build directory + gets its own decompressed copy (copy is nearly as fast as a hardlink). + uncompressed: output.xml, plain copy on restore. ~2.2 G per full build. + Set ITK_WRAP_CACHE_FORMAT=uncompressed to opt in. + +Multi-path cascade (ITK_WRAP_CACHE, colon-separated): + Reads search all paths in order, returning the first hit. Writes go to the + first path that accepts them (atomic rename succeeds). A read-only shared + NFS cache can be listed after the user's writable local SSD cache, e.g.: + export ITK_WRAP_CACHE=/local/ssd/cache:/nfs/lab/shared-cache + Students benefit from the lab cache for L2 hits (saving the full castxml + run) while writing L1 maps only to their own writable local cache. + +Eviction: LRU via background fork after each write (rate-limited to once/60s). + _ok sentinel mtime tracks "last useful" time; oldest entries evicted first. + --evict only touches the first writable cache in the list. + +Environment: + ITK_WRAP_CACHE colon-separated cache roots (default: ~/.cache/itk-wrap) + ITK_WRAP_CACHE_FORMAT storage format: gzip (default) or uncompressed + ITK_WRAP_CACHE_VERBOSE set to 1 for hit/miss logging to stderr + ITK_WRAP_CACHE_BYPASS set to 1 to bypass all caching (same as --no-cache) + ITK_WRAP_CACHE_MAX_SIZE max cache size before LRU eviction (default: 2G) +""" + +import gzip +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time + +# Bump when the key algorithm changes; old entries become unreachable orphans +# (different hash → different L2 path) and are pruned by LRU eviction. +_KEY_VERSION = b"v4\x00" + +# Matches C preprocessor line markers: "# N " (where N is an integer). +# These carry only source-file locations — not C++ semantics — so stripping +# them makes the L2 hash path-independent. +_LINE_MARKER_RE = re.compile(rb"^# \d+ ", re.MULTILINE) + + +def _strip_line_markers(data: bytes) -> bytes: + return b"\n".join( + line for line in data.splitlines() if not _LINE_MARKER_RE.match(line) + ) + + +def _cache_roots(): + """Return ordered list of cache root directories from ITK_WRAP_CACHE. + + The env var is a colon-separated list (like PATH). Reads search all + roots in order; writes go to the first root that accepts them. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if not raw: + return [os.path.join(os.path.expanduser("~"), ".cache", "itk-wrap")] + sep = ";" if sys.platform == "win32" else ":" + return [p for p in raw.split(sep) if p] + + +def _verbose(): + return os.environ.get("ITK_WRAP_CACHE_VERBOSE", "") == "1" + + +def _log(msg): + if _verbose(): + print(f"itk-castxml-cache: {msg}", file=sys.stderr) + + +def _use_uncompressed(): + """Return True when uncompressed storage is explicitly requested via ITK_WRAP_CACHE_FORMAT.""" + return os.environ.get("ITK_WRAP_CACHE_FORMAT", "").lower() in ( + "uncompressed", + "raw", + "plain", + ) + + +def _max_cache_bytes(): + """Parse ITK_WRAP_CACHE_MAX_SIZE (default 2G) into bytes.""" + raw = os.environ.get("ITK_WRAP_CACHE_MAX_SIZE", "2G").strip().upper() + for suffix, mult in ( + ("T", 1 << 40), + ("G", 1 << 30), + ("M", 1 << 20), + ("K", 1 << 10), + ): + if raw.endswith(suffix): + try: + return int(raw[:-1]) * mult + except ValueError: + pass + try: + return int(raw) + except ValueError: + return 2 << 30 # 2 GiB default + + +def _evict_lru(cache_root, max_bytes): + """Remove least-recently-used L2 entries until total is under max_bytes. + + The _ok sentinel mtime is updated on every cache HIT (in _restore_xml), + so it tracks "last time this entry was actually useful to a build." + Entries whose _ok is oldest are evicted first. + """ + l2_root = os.path.join(cache_root, "l2") + if not os.path.isdir(l2_root): + return + + entries = [] + total = 0 + for shard in os.listdir(l2_root): + shard_dir = os.path.join(l2_root, shard) + if not os.path.isdir(shard_dir): + continue + for key in os.listdir(shard_dir): + entry = os.path.join(shard_dir, key) + ok = os.path.join(entry, "_ok") + try: + mtime = os.stat(ok).st_mtime + entry_bytes = sum( + os.path.getsize(os.path.join(entry, fn)) for fn in os.listdir(entry) + ) + entries.append((mtime, entry_bytes, entry)) + total += entry_bytes + except OSError: + pass + + if total <= max_bytes: + return + + _log( + f"evict: {total / 1e9:.2f} GB used, limit {max_bytes / 1e9:.2f} GB" + f" — evicting {len(entries)} candidates (oldest first)" + ) + entries.sort(key=lambda x: x[0]) # ascending mtime → oldest first + + removed = 0 + for _mtime, entry_bytes, entry in entries: + if total <= max_bytes: + break + try: + shutil.rmtree(entry) + total -= entry_bytes + removed += 1 + except OSError: + pass + _log(f"evict: removed {removed} entries, {total / 1e9:.2f} GB remaining") + + +def _evict_async(cache_root): + """Fork a background process to run LRU eviction without blocking the build. + + Rate-limited via a _evict_ts sentinel: won't re-check within 60 seconds. + This prevents all 816 parallel build workers from checking simultaneously. + """ + sentinel = os.path.join(cache_root, "_evict_ts") + try: + if time.time() - os.stat(sentinel).st_mtime < 60: + return + except OSError: + pass + try: + open(sentinel, "w").close() # noqa: WPS515 + except OSError: + return + if sys.platform == "win32": + return + pid = os.fork() + if pid == 0: + try: + _evict_lru(cache_root, _max_cache_bytes()) + finally: + os._exit(0) + + +def _bypass_mode(): + """Return True when caching should be skipped entirely. + + Controlled by --no-cache as the first positional arg (before castxml binary) + or by ITK_WRAP_CACHE_BYPASS=1 in the environment. Use for single-use builds + where the castxml -E overhead would cost more than the cache saves. + """ + return os.environ.get("ITK_WRAP_CACHE_BYPASS", "") == "1" + + +def _parse_args(argv): + """ + Parse a castxml command line into structured components. + + Strips a leading --no-cache flag (before the castxml binary) when present. + Returns (castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache) + where passthrough_flags preserves the original ordering for subprocess use. + """ + no_cache = False + if argv and argv[0] == "--no-cache": + no_cache = True + argv = argv[1:] + + if not argv: + return None, None, None, None, [], no_cache + + castxml_bin = argv[0] + output_xml = None + inc_file = None + cxx_file = None + passthrough_flags = [] + + i = 1 + while i < len(argv): + arg = argv[i] + if arg == "-o" and i + 1 < len(argv): + output_xml = argv[i + 1] + # Include in passthrough so castxml writes its output normally + passthrough_flags.extend([arg, argv[i + 1]]) + i += 2 + elif arg.startswith("@"): + inc_file = arg[1:] + passthrough_flags.append(arg) + i += 1 + elif arg.endswith(".cxx"): + cxx_file = arg + passthrough_flags.append(arg) + i += 1 + else: + passthrough_flags.append(arg) + i += 1 + + return castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache + + +def _castxml_content_hash(castxml_bin, primary_root): + """Return a stable SHA-256 of the castxml binary, cached on disk. + + Sidecar file stores "size mtime_ns sha256" so re-hashing only happens when + size or mtime changes. After `ninja -t clean`, castxml is re-linked with + the same content → same hash → stable L1 key → L1 hits on warm rebuilds. + Sidecar lives in the first (writable) cache root. + """ + try: + st = os.stat(castxml_bin) + except OSError: + return "missing" + + # One sidecar per binary path (path key avoids slashes in filename) + path_key = hashlib.sha256(castxml_bin.encode()).hexdigest()[:16] + sidecar = os.path.join(primary_root, "_binhash", path_key) + + try: + with open(sidecar) as f: + parts = f.read().split() + if ( + len(parts) == 3 + and int(parts[0]) == st.st_size + and int(parts[1]) == st.st_mtime_ns + ): + return parts[2] + except (OSError, ValueError): + pass + + # Sidecar miss or stale — hash the binary (one-time cost per unique binary) + h = hashlib.sha256() + try: + with open(castxml_bin, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + except OSError: + return "unreadable" + + content_hash = h.hexdigest() + try: + os.makedirs(os.path.dirname(sidecar), exist_ok=True) + tmp = sidecar + ".tmp" + with open(tmp, "w") as f: + f.write(f"{st.st_size} {st.st_mtime_ns} {content_hash}\n") + os.rename(tmp, sidecar) + except OSError: + pass + return content_hash + + +def _l1_key(castxml_bin, inc_file, cxx_file, passthrough_flags, primary_root): + """Compute L1 cache key from direct inputs only (~0.2s, no subprocess).""" + h = hashlib.sha256() + + # Stable content fingerprint — survives re-link with unchanged binary. + h.update( + f"castxml\x00{_castxml_content_hash(castxml_bin, primary_root)}\x00".encode() + ) + + # Response file: include dirs + defines passed via @file + if inc_file: + try: + with open(inc_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00inc_miss\x00") + h.update(b"\x00") + + # Input .cxx source + if cxx_file: + try: + with open(cxx_file, "rb") as f: + h.update(f.read()) + except OSError: + h.update(b"\x00cxx_miss\x00") + h.update(b"\x00") + + # Remaining flags (--castxml-cc-gnu, compiler path, std flags, etc.) + # Skip -o and the output xml path — irrelevant to content. + skip_next = False + for flag in passthrough_flags: + if skip_next: + skip_next = False + continue + if flag == "-o": + skip_next = True + continue + if flag.endswith(".xml"): + continue + h.update(flag.encode()) + h.update(b"\x00") + + return h.hexdigest() + + +def _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_output): + """ + Build a `castxml -E` command that preprocesses the same inputs. + + Strips --castxml-output, --castxml-start (XML-generation flags), + replaces -o with the temp preprocess output path, appends -E. + """ + cmd = [castxml_bin] + skip_next = False + for arg in passthrough_flags: + if skip_next: + skip_next = False + continue + if arg.startswith("--castxml-output"): + continue + if arg.startswith("--castxml-start"): + if "=" not in arg: + skip_next = True + continue + if arg == "-o": + skip_next = True # drop -o and its value (xml output path) + continue + if arg.endswith(".xml"): + continue + cmd.append(arg) + cmd.extend(["-E", "-o", pre_output]) + return cmd + + +def _compute_l2_key(castxml_bin, passthrough_flags): + """ + Run castxml -E and hash the normalised preprocessed output. + + L1 key is NOT mixed in: two build directories with identical source produce + identical L2 keys and share cache entries regardless of castxml binary mtime. + Returns the L2 key string, or None if preprocessing fails. + """ + with tempfile.NamedTemporaryFile(suffix=".i", delete=False) as tmp: + pre_path = tmp.name + + try: + cmd = _build_preprocess_cmd(castxml_bin, passthrough_flags, pre_path) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + _log(f"castxml -E failed (exit {result.returncode})") + return None + h = hashlib.sha256(_KEY_VERSION) + with open(pre_path, "rb") as f: + h.update(_strip_line_markers(f.read())) + return h.hexdigest() + except OSError: + return None + finally: + try: + os.unlink(pre_path) + except OSError: + pass + + +def _l1_file(cache_root, l1_key): + return os.path.join(cache_root, "l1", l1_key[:2], l1_key, "l2_key") + + +def _l2_dir(cache_root, l2_key): + return os.path.join(cache_root, "l2", l2_key[:2], l2_key) + + +def _restore_xml(cache_root, l2_key, output_xml): + """Restore cached XML to output_xml from one cache root. Returns True on success.""" + entry = _l2_dir(cache_root, l2_key) + ok_file = os.path.join(entry, "_ok") + xml_plain = os.path.join(entry, "output.xml") + xml_gz = os.path.join(entry, "output.xml.gz") + + if not os.path.isfile(ok_file): + return False + + try: + os.makedirs(os.path.dirname(os.path.abspath(output_xml)), exist_ok=True) + + if os.path.isfile(xml_plain): + shutil.copy2(xml_plain, output_xml) + elif os.path.isfile(xml_gz): + with gzip.open(xml_gz, "rb") as src, open(output_xml, "wb") as dst: + shutil.copyfileobj(src, dst) + else: + return False + + # Touch _ok to record "last useful" time for LRU eviction. + os.utime(ok_file, None) + return True + except OSError: + return False + + +def _restore_from_caches(roots, l2_key, output_xml): + """Search all cache roots for l2_key, restore on first hit.""" + for root in roots: + if _restore_xml(root, l2_key, output_xml): + return root # return the root that had the hit + return None + + +def _store(roots, l1_key, l2_key, output_xml): + """Store output_xml to L2 cache and write L1→L2 mapping atomically. + + Tries each root in order and writes to the first that accepts an atomic + rename. Read-only roots (e.g. a shared NFS lab cache) are silently + skipped. + """ + uncompressed = _use_uncompressed() + + # L2 entry — write to first writable root + write_root = None + for root in roots: + entry = _l2_dir(root, l2_key) + tmp = entry + ".tmp" + try: + if os.path.exists(tmp): + shutil.rmtree(tmp) + os.makedirs(tmp, exist_ok=True) + + if os.path.isfile(output_xml): + if uncompressed: + shutil.copy2(output_xml, os.path.join(tmp, "output.xml")) + else: + with ( + open(output_xml, "rb") as src, + gzip.open( + os.path.join(tmp, "output.xml.gz"), "wb", compresslevel=6 + ) as dst, + ): + shutil.copyfileobj(src, dst) + + with open(os.path.join(tmp, "_meta.json"), "w") as f: + json.dump( + { + "l1_key": l1_key, + "l2_key": l2_key, + "format": "uncompressed" if uncompressed else "gzip", + }, + f, + ) + open(os.path.join(tmp, "_ok"), "w").close() # noqa: WPS515 + + if os.path.exists(entry): + shutil.rmtree(entry) + os.rename(tmp, entry) + write_root = root + break + except OSError as exc: + _log(f"L2 store failed for {root}: {exc}") + shutil.rmtree(tmp, ignore_errors=True) + + if write_root is None: + _log("L2 store failed in all cache roots — no entry written") + return + + # L1→L2 mapping — write to same root that accepted the L2 entry + l1f = _l1_file(write_root, l1_key) + try: + os.makedirs(os.path.dirname(l1f), exist_ok=True) + with open(l1f + ".tmp", "w") as f: + f.write(l2_key) + os.rename(l1f + ".tmp", l1f) + except OSError as exc: + _log(f"L1 map store failed: {exc}") + + _evict_async(write_root) + + +def _store_l1_mapping(roots, l1_key, l2_key): + """Write an L1→L2 mapping to the first writable root (no L2 write).""" + for root in roots: + l1f = _l1_file(root, l1_key) + try: + os.makedirs(os.path.dirname(l1f), exist_ok=True) + with open(l1f + ".tmp", "w") as f: + f.write(l2_key) + os.rename(l1f + ".tmp", l1f) + return + except OSError: + continue + + +def _run_castxml(castxml_bin, passthrough_flags): + result = subprocess.run([castxml_bin] + passthrough_flags) + return result.returncode + + +def main(): + argv = sys.argv[1:] + if not argv: + print(__doc__, file=sys.stderr) + return 1 + + # Stand-alone eviction subcommand: python3 itk-castxml-cache.py --evict + if argv[0] == "--evict": + import argparse + + p = argparse.ArgumentParser(prog="itk-castxml-cache.py --evict") + p.add_argument( + "--max-size", + default=None, + help="Max cache size before eviction (e.g. 1G, 500M)", + ) + p.add_argument( + "--cache-dir", default=None, help="Cache root (overrides ITK_WRAP_CACHE)" + ) + args = p.parse_args(argv[1:]) + roots = [args.cache_dir] if args.cache_dir else _cache_roots() + if args.max_size: + os.environ["ITK_WRAP_CACHE_MAX_SIZE"] = args.max_size + _evict_lru(roots[0], _max_cache_bytes()) + return 0 + + castxml_bin, output_xml, inc_file, cxx_file, passthrough_flags, no_cache = ( + _parse_args(argv) + ) + + if no_cache or _bypass_mode() or not output_xml or not cxx_file: + return _run_castxml(castxml_bin, passthrough_flags) + + roots = _cache_roots() + # primary_root is used for binary sidecar storage; writes go to first writable + primary_root = roots[0] + + # ── L1 check (fast, no subprocess) ────────────────────────────────────── + l1_key = _l1_key(castxml_bin, inc_file, cxx_file, passthrough_flags, primary_root) + + stored_l2_key = None + for root in roots: + l1f = _l1_file(root, l1_key) + if os.path.isfile(l1f): + try: + with open(l1f) as f: + stored_l2_key = f.read().strip() or None + if stored_l2_key: + break + except OSError: + pass + + # ── L1 HIT: skip castxml -E entirely ──────────────────────────────────── + if stored_l2_key is not None: + hit_root = _restore_from_caches(roots, stored_l2_key, output_xml) + if hit_root is not None: + _log(f"HIT {os.path.basename(cxx_file)} (l1→l2={stored_l2_key[:8]})") + return 0 + # L2 entry missing or corrupt despite L1 hit — fall through to -E check. + _log(f"L2 entry missing for {cxx_file}, re-running castxml -E") + + # ── castxml -E to compute actual L2 key (L1 miss or L2 corrupt) ───────── + actual_l2_key = _compute_l2_key(castxml_bin, passthrough_flags) + + if actual_l2_key is None: + _log(f"preprocess failed for {cxx_file} — passing through") + return _run_castxml(castxml_bin, passthrough_flags) + + # ── L2 store lookup (handles cross-dir hits: L1 miss, L2 populated) ───── + hit_root = _restore_from_caches(roots, actual_l2_key, output_xml) + if hit_root is not None: + _log(f"HIT {os.path.basename(cxx_file)} (l2={actual_l2_key[:8]})") + # Populate L1 map so the next rebuild skips castxml -E + _store_l1_mapping(roots, l1_key, actual_l2_key) + return 0 + + # ── Full castxml run ───────────────────────────────────────────────────── + _log(f"MISS {os.path.basename(cxx_file)}") + try: + os.unlink(output_xml) + except OSError: + pass + rc = _run_castxml(castxml_bin, passthrough_flags) + if rc == 0: + _store(roots, l1_key, actual_l2_key, output_xml) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Wrapping/Generators/Python/itk/pyi_generator.py b/Wrapping/Generators/Python/itk/pyi_generator.py index d8007fb6aed..7ba2ccf6cc3 100644 --- a/Wrapping/Generators/Python/itk/pyi_generator.py +++ b/Wrapping/Generators/Python/itk/pyi_generator.py @@ -5,6 +5,8 @@ """ import os +import sys +import sqlite3 from os import remove from argparse import ArgumentParser from io import StringIO @@ -14,6 +16,27 @@ import re from collections import defaultdict +PKL_DB_SCHEMA_VERSION = 1 + + +def _pkl_db_path(fallback_dir: str = "") -> Path: + """Return the pkl DB path. + + Priority: + 1. $ITK_WRAP_CACHE (first path in colon/semicolon-separated list) + 2. fallback_dir (build-tree itk-pkl/ directory passed via --pkl_dir) + + Developers must explicitly set ITK_WRAP_CACHE to opt into a shared or + home-directory location; the build tree is the default. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if raw: + sep = ";" if sys.platform == "win32" else ":" + cache_root = raw.split(sep)[0] + else: + cache_root = fallback_dir + return Path(cache_root) / f"itk-pkl-v{PKL_DB_SCHEMA_VERSION}.db" + # The ITKClass is duplicated in igenerator.py class ITKClass: @@ -379,13 +402,17 @@ def write_class_proxy_pyi( pyi_file.write(interfaces_code) -def unpack(file_names: [str], save_dir: str) -> str | None: +def unpack(keys: list[str], db_conn: sqlite3.Connection, save_dir: str) -> str | None: class_definitions = [] - for file_name in file_names: - with open(file_name, "rb") as pickled_file: - itk_class = pickle.load(pickled_file) - class_definitions.append(itk_class) + for key in keys: + row = db_conn.execute( + "SELECT data FROM pkl_data WHERE key=?", (key,) + ).fetchone() + if row is None: + print(f"WARNING: pkl key {key!r} not found in database, skipping.") + continue + class_definitions.append(pickle.loads(row[0])) base = merge(class_definitions) @@ -556,7 +583,6 @@ def merge(class_definitions: []) -> ITKClass | None: remove(invalid_index_file) for missing_index_file in missing_index_files: - # continue on without missing file, display warning print( f"WARNING: index file {missing_index_file} is missing, " f"Python stub hints will not be generated for this file. " @@ -568,52 +594,38 @@ def merge(class_definitions: []) -> ITKClass | None: print(f"Exception: {except_comment}") raise Exception(except_comment) - indexed_pickled_files = set() + # Collect DB keys from all .index.txt manifests (keys, not file paths). + indexed_pkl_keys: set[str] = set() for index_file in sorted(index_files): with open(index_file) as file: for line in file: - indexed_pickled_files.add(line.strip()) - - existing_pickled_files = { - filepath.replace(os.sep, "/") - for filepath in glob.glob(f"{options.pkl_dir}/*.pkl") - } + key = line.strip() + if key: + indexed_pkl_keys.add(key) - invalid_pickled_files = existing_pickled_files - indexed_pickled_files - missing_pickled_files = indexed_pickled_files - existing_pickled_files + if len(indexed_pkl_keys) == 0: + raise Exception(f"No pkl keys were found in index files in '{options.pkl_dir}'") - if options.debug_code: - for invalid_pickle_file in invalid_pickled_files: - print( - f"WARNING: Outdated pickle file {invalid_pickle_file} has been removed" - ) - remove(invalid_pickle_file) - - for missing_file in missing_pickled_files: - # continue on without missing file, display warning - print( - f"WARNING: pickle file {missing_file} is missing, Python stub hints will not be generated for this file." - ) - indexed_pickled_files.remove(missing_file) - - if len(indexed_pickled_files) == 0: - raise Exception(f"No pickle files were found in directory {options.pkl_dir}") - - indexed_pickled_files = sorted(list(indexed_pickled_files)) + indexed_pkl_keys_sorted = sorted(indexed_pkl_keys) output_template_import_list: list[str] = [] output_proxy_import_list: list[str] = [] - class_name_dict = defaultdict(list) - for file in indexed_pickled_files: - current_class_name = re.split(r"\.", PurePath(file).parts[-1])[0] - class_name_dict[current_class_name].append(file) + class_name_dict: defaultdict[str, list[str]] = defaultdict(list) + for key in indexed_pkl_keys_sorted: + current_class_name = key.split(".")[0] + class_name_dict[current_class_name].append(key) - for current_class_name, class_files in class_name_dict.items(): - class_name = unpack(class_files, options.pyi_dir) + db_conn = sqlite3.connect(_pkl_db_path(options.pkl_dir)) + db_conn.execute("PRAGMA journal_mode=WAL") + + for current_class_name, class_keys in class_name_dict.items(): + class_name = unpack(class_keys, db_conn, options.pyi_dir) if class_name is not None: output_template_import_list.append(f"from .{class_name}Template import *\n") output_proxy_import_list.append(f"from .{class_name}Proxy import *\n") + db_conn.close() + output_init_import_file = init_init_import_file() output_proxy_import_file = init_proxy_import_file() diff --git a/Wrapping/Generators/SwigInterface/igenerator.py b/Wrapping/Generators/SwigInterface/igenerator.py index bcda8695eab..1121371f993 100755 --- a/Wrapping/Generators/SwigInterface/igenerator.py +++ b/Wrapping/Generators/SwigInterface/igenerator.py @@ -57,9 +57,12 @@ # -*- coding: utf-8 -*- import collections import pickle +import shutil +import sqlite3 import sys import os import re +import time from argparse import ArgumentParser from io import StringIO from os.path import exists @@ -68,6 +71,43 @@ from typing import Any import logging +PKL_DB_SCHEMA_VERSION = 1 + + +def _pkl_db_path(fallback_dir: str = "") -> Path: + """Return the pkl DB path. + + Priority: + 1. $ITK_WRAP_CACHE (first path in colon/semicolon-separated list) + 2. fallback_dir (build-tree itk-pkl/ directory passed via --pkl_dir) + + Developers must explicitly set ITK_WRAP_CACHE to opt into using a shared + or home-directory location; the build tree is the default. + """ + raw = os.environ.get("ITK_WRAP_CACHE", "") + if raw: + sep = ";" if sys.platform == "win32" else ":" + cache_root = raw.split(sep)[0] + else: + cache_root = fallback_dir + return Path(cache_root) / f"itk-pkl-v{PKL_DB_SCHEMA_VERSION}.db" + + +def _open_pkl_db(pkl_dir: str) -> sqlite3.Connection: + db_path = _pkl_db_path(pkl_dir) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute( + "CREATE TABLE IF NOT EXISTS pkl_data (key TEXT PRIMARY KEY, data BLOB NOT NULL)" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS manifest (module TEXT NOT NULL, key TEXT NOT NULL)" + ) + conn.execute("CREATE INDEX IF NOT EXISTS manifest_module ON manifest(module)") + conn.commit() + return conn + def argument_parser(): cmdln_arg_parser = ArgumentParser() @@ -205,7 +245,24 @@ def argument_parser(): dest="pkl_dir", default="", type=str, - help="The directory for .pyi files to be generated", + help="Directory for per-submodule .index.txt manifests and per-module stamp files.", + ) + cmdln_arg_parser.add_argument( + "--module_name", + action="store", + dest="module_name", + default="", + type=str, + help="ITK module name (e.g. ITKCommon); used as the manifest key in the pkl DB.", + ) + cmdln_arg_parser.add_argument( + "--pkl_stamp", + action="store", + dest="pkl_stamp", + default="", + type=str, + help="Stamp file touched after the pkl DB transaction commits; " + "declared as a CMake OUTPUT so ninja re-runs igenerator when stale.", ) cmdln_arg_parser.add_argument( "-d", @@ -220,7 +277,15 @@ def argument_parser(): glb_options = argument_parser() sys.path.insert(1, glb_options.pygccxml_path) -import pygccxml # noqa: E402 +pygccxml = None # populated lazily by _load_pygccxml() on cache miss + + +def _load_pygccxml(): + """Import pygccxml into the module namespace; called only on cache miss.""" + global pygccxml + import importlib + + pygccxml = importlib.import_module("pygccxml") # Global debugging variables @@ -1901,7 +1966,9 @@ def get_submodule_namespace( def generate_swig_input( submodule_name, - pkl_dir, + pkl_db_rows, + manifest_rows, + module_name, pygccxml_config, options, snake_case_process_object_functions, @@ -1924,45 +1991,25 @@ def generate_swig_input( swig_input_generator.snakeCaseProcessObjectFunctions ) - # Write index list of generated .pkl files + # Collect pkl data for the SQLite batch write; write key manifest to .index.txt index_file_contents: StringIO = StringIO() all_keys = swig_input_generator.classes.keys() if len(all_keys): for itk_class in all_keys: - # Future problem will be that a few files will be empty - # Can either somehow detect this or accept it - # pickle class here class_name: str = swig_input_generator.classes[itk_class].class_name submodule_name: str = swig_input_generator.classes[itk_class].submodule_name - pickled_filename: str = f"{pkl_dir}/{class_name}.{submodule_name}.pkl" - - # Only write to the pickle file if it does not match what is already saved. - overwrite: bool = False - pickle_exists: bool = exists(pickled_filename) - if pickle_exists: - with open(pickled_filename, "rb") as pickled_file: - existing_itk_class = pickle.load(pickled_file) - overwrite = not ( - existing_itk_class == swig_input_generator.classes[itk_class] - ) - if overwrite or not pickle_exists: - with open(pickled_filename, "wb") as pickled_file: - pickle.dump(swig_input_generator.classes[itk_class], pickled_file) - - index_file_contents.write(pickled_filename + "\n") + key: str = f"{class_name}.{submodule_name}" + pkl_db_rows.append( + (key, pickle.dumps(swig_input_generator.classes[itk_class])) + ) + manifest_rows.append((module_name, key)) + index_file_contents.write(key + "\n") else: - # The following warning is useful for debugging, and eventually we - # may wish to find a way to remove modules that are not currently part - # of the build. For example, currently all *.wrap files are processed and listed - # as module dependencies. If FFTW is not enabled, that causes empty submodules - # to be created as dependencies unnecessarily. - # Changing that behavior will require structural code changes, or alternate - # mechanisms to be implemented. if glb_options.debug_code: print( f"WARNING: {submodule_name} has no classes identified, but was listed as a dependent submodule." ) - generate_pyi_index_files(submodule_name, index_file_contents, pkl_dir) + generate_pyi_index_files(submodule_name, index_file_contents, options.pkl_dir) def main(): @@ -1971,7 +2018,9 @@ def main(): raise ValueError(f"Required directory missing '{options.pyi_dir}'") if options.pkl_dir == "": - raise ValueError(f"Required directory missing '{options.pkl_dir}'") + raise ValueError("--pkl_dir is required for .index.txt manifests") + if options.module_name == "": + raise ValueError("--module_name is required for the pkl DB manifest table") # Ensure that the requested stub file directory exists if options.pyi_dir != "": @@ -1979,18 +2028,6 @@ def main(): if options.pkl_dir != "": Path(options.pkl_dir).mkdir(exist_ok=True, parents=True) - # init the pygccxml stuff - pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) - pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False - pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True - - pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( - xml_generator_path=options.castxml_path, - xml_generator="castxml", - # Use castxml-output=1 to take advantage of the newer XML format - flags=["--castxml-output=1"], - ) - submodule_names_list: list[str] = [] # The first mdx file is the master index file for this module. master_mdx_filename: Path = Path(options.mdx[0]) @@ -2011,6 +2048,18 @@ def main(): if submodule_name not in submodule_names_list: submodule_names_list.append(submodule_name) + _load_pygccxml() + + pygccxml.utils.loggers.cxx_parser.setLevel(logging.CRITICAL) + pygccxml.declarations.scopedef_t.RECURSIVE_DEFAULT = False + pygccxml.declarations.scopedef_t.ALLOW_EMPTY_MDECL_WRAPPER = True + + pygccxml_config = pygccxml.parser.config.xml_generator_configuration_t( + xml_generator_path=options.castxml_path, + xml_generator="castxml", + flags=["--castxml-output=1"], + ) + for submodule_name in submodule_names_list: wrappers_namespace: Any = global_submodule_cache.get_submodule_namespace( submodule_name, options.library_output_dir, pygccxml_config @@ -2032,15 +2081,27 @@ def main(): ordered_submodule_list.append(submodule_name) del submodule_names_list + pkl_db_rows: list[tuple[str, bytes]] = [] + manifest_rows: list[tuple[str, str]] = [] for submodule_name in ordered_submodule_list: generate_swig_input( submodule_name, - options.pkl_dir, + pkl_db_rows, + manifest_rows, + options.module_name, pygccxml_config, options, snake_case_process_object_functions, ) + if pkl_db_rows: + conn = _open_pkl_db(options.pkl_dir) + with conn: + conn.execute("DELETE FROM manifest WHERE module=?", (options.module_name,)) + conn.executemany("INSERT OR REPLACE INTO pkl_data VALUES(?,?)", pkl_db_rows) + conn.executemany("INSERT INTO manifest VALUES(?,?)", manifest_rows) + conn.close() + snake_case_file = options.snake_case_file if len(snake_case_file) > 1: with open(snake_case_file, "w") as ff: @@ -2054,6 +2115,9 @@ def main(): ff.write("'" + function + "', ") ff.write(")\n") + if options.pkl_stamp: + Path(options.pkl_stamp).touch() + if __name__ == "__main__": main() diff --git a/Wrapping/macro_files/itk_auto_load_submodules.cmake b/Wrapping/macro_files/itk_auto_load_submodules.cmake index 4ca246ebc1e..1a0edec86ba 100644 --- a/Wrapping/macro_files/itk_auto_load_submodules.cmake +++ b/Wrapping/macro_files/itk_auto_load_submodules.cmake @@ -199,12 +199,33 @@ function(generate_castxml_commandline_flags) endforeach() # ===== Run the castxml command + if( + ITK_WRAP_CASTXML_CACHE + AND + Python3_EXECUTABLE + AND + ITK_WRAP_CASTXML_CACHE_SCRIPT + ) + set( + _castxml_cmd + ${Python3_EXECUTABLE} + "${ITK_WRAP_CASTXML_CACHE_SCRIPT}" + ${CASTXML_EXECUTABLE} + ) + list(APPEND _castxml_depends "${ITK_WRAP_CASTXML_CACHE_SCRIPT}") + else() + set( + _castxml_cmd + ${_ccache_cmd} + ${CASTXML_EXECUTABLE} + ) + endif() add_custom_command( OUTPUT ${xml_file} COMMAND - ${_build_env} ${_ccache_cmd} ${CASTXML_EXECUTABLE} -o ${xml_file} - --castxml-output=1 ${_target} --castxml-start _wrapping_ ${_castxml_cc} -w + ${_build_env} ${_castxml_cmd} -o ${xml_file} --castxml-output=1 ${_target} + --castxml-start _wrapping_ ${_castxml_cc} -w -c # needed for ccache to think we are not calling for link @${castxml_inc_file} ${cxx_file} VERBATIM @@ -214,6 +235,7 @@ function(generate_castxml_commandline_flags) ${castxml_inc_file} ${_hdrs} ) + unset(_castxml_cmd) unset(cxx_file) unset(castxml_inc_file) unset(_build_env) diff --git a/Wrapping/macro_files/itk_end_wrap_module.cmake b/Wrapping/macro_files/itk_end_wrap_module.cmake index 04bde56dc4b..231582a1018 100644 --- a/Wrapping/macro_files/itk_end_wrap_module.cmake +++ b/Wrapping/macro_files/itk_end_wrap_module.cmake @@ -161,11 +161,13 @@ macro(itk_end_wrap_module) # Set up outputs and byproducts for custom command set(igenerator_outputs "") set(igenerator_byproducts "") + set(pkl_stamp_file "${ITK_PKL_DIR}/${WRAPPER_LIBRARY_NAME}.stamp") list(APPEND igenerator_outputs "${i_files}") # Typedefs/.i list(APPEND igenerator_outputs "${typedef_files}") # Typedefs/SwigInterface.h list(APPEND igenerator_outputs "${idx_files}") # Typedefs/.idx list(APPEND igenerator_outputs "${snake_case_config_file}") # Generators/Python/itk/Configuration/_snake_case.py + list(APPEND igenerator_outputs "${pkl_stamp_file}") # itk-pkl/.stamp if(CMAKE_GENERATOR STREQUAL "Ninja") # Ninja generator requires byproduct for correct dependency handling # See https://cmake.org/cmake/help/latest/policy/CMP0058.html @@ -188,7 +190,8 @@ macro(itk_end_wrap_module) --library-output-dir "${WRAPPER_LIBRARY_OUTPUT_DIR}" --submodule-order "${THIS_MODULE_SUBMODULE_ORDER}" --pyi_index_list "${ITK_PYI_INDEX_FILES}" --pyi_dir "${ITK_STUB_DIR}" --pkl_dir - "${ITK_PKL_DIR}" + "${ITK_PKL_DIR}" --module_name "${WRAPPER_LIBRARY_NAME}" --pkl_stamp + "${pkl_stamp_file}" DEPENDS ${IGENERATOR} ${ITK_WRAP_DOC_DOCSTRING_FILES} @@ -202,6 +205,7 @@ macro(itk_end_wrap_module) VERBATIM ) + unset(pkl_stamp_file) unset(snake_case_config_file) else() #message(FATAL_ERROR "Number of interface files is 0 :${WRAPPER_LIBRARY_NAME}:") diff --git a/pixi.lock b/pixi.lock index d476f4e4cce..0bea1db9e08 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1592,6 +1592,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.11.0-h4d9bdce_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/castxml-0.7.0-hde8d07d_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ccache-4.13.6-hedf47ba_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.1.2-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-14.3.0-hb991d5c_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.11.0-hfcd1e18_0.conda @@ -1618,6 +1619,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-hcd61629_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-h767d61c_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.3.0-h5888daf_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-38_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libllvm20-20.1.8-hf7376ad_1.conda @@ -1645,6 +1647,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/rhash-1.4.6-hb9d3cd8_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/swig-4.4.1-h7a96c5f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.3-hb47aa4a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb8e6e7a_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1671,6 +1674,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-compiler-1.11.0-hdceaead_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/castxml-0.7.0-ha3e84ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ccache-4.13.6-h185addb_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.1.2-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/conda-gcc-specs-14.3.0-h92dcf8a_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cxx-compiler-1.11.0-h7b35c40_0.conda @@ -1697,6 +1701,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran-15.2.0-he9431aa_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgfortran5-15.2.0-h87db57e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libgomp-15.2.0-he277a41_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libhiredis-1.3.0-h5ad3122_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/liblapack-3.9.0-38_h88aeb00_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libllvm20-20.1.8-hfd2ba90_1.conda @@ -1724,6 +1729,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rhash-1.4.6-h86ecc28_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/swig-4.4.1-h512d76c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/tk-8.6.13-noxft_h5688188_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/xxhash-0.8.3-hd794028_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/zstd-1.5.7-hbcf94c1_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda @@ -1759,6 +1765,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-compiler-1.11.0-h7a00415_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/castxml-0.7.0-hb171174_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ccache-4.13.6-h894318c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools_osx-64-1024.3-llvm19_1_h3b512aa_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/clang-19-19.1.7-default_hc369343_5.conda @@ -1787,6 +1794,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.5.2-h750e83c_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran-15.2.0-h306097a_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libgfortran5-15.2.0-h336fb69_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libhiredis-1.3.0-h240833e_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/liblapack-3.9.0-38_h859234e_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libllvm19-19.1.7-h56e7563_2.conda @@ -1816,6 +1824,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/swig-4.4.1-hdac4ec2_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tapi-1300.6.5-h390ca13_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-hf689a15_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/xxhash-0.8.3-h13e91ac_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h8210216_2.conda osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-hbd8a1cb_0.conda @@ -1835,6 +1844,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-compiler-1.11.0-h61f9b84_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/castxml-0.7.0-hfc9ce51_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ccache-4.13.6-h414bf82_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools_osx-arm64-1024.3-llvm19_1_h8c76c84_9.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clang-19-19.1.7-default_h73dfc95_5.conda @@ -1864,6 +1874,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-he5f378a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-hfcf01ff_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-h742603c_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libhiredis-1.3.0-h286801f_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.9.0-38_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libllvm19-19.1.7-h8e0c9ce_2.conda @@ -1893,6 +1904,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/swig-4.4.1-h4366dc5_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tapi-1300.6.5-h03f4b80_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xxhash-0.8.3-haa4e116_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-h6491c7d_2.conda win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.10.5-h4c7d964_0.conda @@ -1911,6 +1923,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/win-64/castxml-0.7.0-ha22e26b_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ccache-4.13.6-h7fd822b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.1.2-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cxx-compiler-1.11.0-h1c1089f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda @@ -1922,6 +1935,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/libgcc-15.2.0-h1383e82_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgfortran5-15.2.0-hf2bee02_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libgomp-15.2.0-h1383e82_7.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libhiredis-1.3.0-he0c23c2_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.9.0-38_hf9ab0e9_mkl.conda @@ -1950,6 +1964,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_32.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vs2022_win-64-19.44.35207-ha74f236_32.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/xxhash-0.8.3-hbba6f48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-hbeecb71_2.conda packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2211,6 +2226,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 855698 timestamp: 1777926446042 - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.0.0-py313hf46b229_1.conda @@ -3206,6 +3222,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140759 timestamp: 1748219397797 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda @@ -4283,6 +4302,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108219 timestamp: 1746457673761 - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda @@ -4560,6 +4582,7 @@ packages: - zstd >=1.5.7,<1.6.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 843745 timestamp: 1777926452238 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-2.0.0-py313h897158f_1.conda @@ -5499,6 +5522,9 @@ packages: - libstdcxx >=13 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 140290 timestamp: 1748220539026 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/libiconv-1.18-h90929bb_2.conda @@ -6499,6 +6525,9 @@ packages: - libgcc >=13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105762 timestamp: 1746457675564 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/yaml-0.2.5-h80f16a2_3.conda @@ -7376,6 +7405,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 657026 timestamp: 1777926755291 - conda: https://conda.anaconda.org/conda-forge/osx-64/cctools-1024.3-h67a6458_9.conda @@ -8299,6 +8329,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 59830 timestamp: 1748219625377 - conda: https://conda.anaconda.org/conda-forge/osx-64/libiconv-1.18-h57a12c2_2.conda @@ -9048,6 +9081,9 @@ packages: - __osx >=10.13 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 108449 timestamp: 1746457796808 - conda: https://conda.anaconda.org/conda-forge/osx-64/yaml-0.2.5-h4132b18_3.conda @@ -9210,6 +9246,7 @@ packages: - libhiredis >=1.3.0,<1.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 601285 timestamp: 1777926636412 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cctools-1024.3-hd01ab73_9.conda @@ -10143,6 +10180,9 @@ packages: - libcxx >=18 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 56746 timestamp: 1748219528586 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libiconv-1.18-h23cfdf5_2.conda @@ -10893,6 +10933,9 @@ packages: - __osx >=11.0 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 98913 timestamp: 1746457827085 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda @@ -11048,6 +11091,7 @@ packages: - xxhash >=0.8.3,<0.8.4.0a0 license: GPL-3.0-only license_family: GPL + run_exports: {} size: 692199 timestamp: 1777926529520 - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-2.0.0-py313h5ea7bf4_1.conda @@ -11530,6 +11574,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-3-Clause license_family: BSD + run_exports: + weak: + - libhiredis >=1.3.0,<1.4.0a0 size: 64205 timestamp: 1748219812303 - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.1-default_h64bd3f2_1002.conda @@ -12394,6 +12441,9 @@ packages: - vc14_runtime >=14.29.30139 license: BSD-2-Clause license_family: BSD + run_exports: + weak: + - xxhash >=0.8.3,<0.8.4.0a0 size: 105768 timestamp: 1746458183583 - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda diff --git a/pyproject.toml b/pyproject.toml index 2fe4d3b2644..1faed5b693d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ libopenblas = ">=0.3.30,<0.4" libgfortran5 = ">=15.2.0,<16" swig = ">=4.4.1" castxml = ">=0.7.0,<0.8" +ccache = ">=4.13.6,<5" [tool.pixi.feature.cxx.tasks.configure] cmd = '''cmake \ @@ -236,10 +237,58 @@ cmd = '''cmake \ -DITK_USE_SYSTEM_SWIG:BOOL=ON \ -DITK_USE_SYSTEM_CASTXML:BOOL=ON \ -DCMAKE_BUILD_TYPE:STRING=Release \ - -DBUILD_TESTING:BOOL=ON''' + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' description = "Configure ITK Python" outputs = ["build-python/CMakeFiles/"] +[tool.pixi.feature.python.tasks.configure-python-ci] +cmd = '''cmake \ + -Bbuild-python \ + -S. \ + -GNinja \ + -DITK_WRAP_PYTHON:BOOL=ON \ + -DITK_WRAP_CASTXML_CACHE:BOOL=ON \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache''' +description = "Configure ITK Python for CI (castxml cache enabled; set ITK_WRAP_CACHE before building)" +outputs = ["build-python/CMakeFiles/"] + +[tool.pixi.feature.python.tasks.build-python-ci] +cmd = "cmake --build build-python" +description = "Build ITK Python for CI" +outputs = ["build-python/lib/**"] +depends-on = ["configure-python-ci"] + +[tool.pixi.feature.python.tasks.test-python-ci] +cmd = "ctest -j3 --test-dir build-python --output-on-failure" +description = "Test ITK Python for CI" +depends-on = ["build-python-ci"] + +[tool.pixi.feature.python.tasks.configure-python-local] +cmd = '''cmake \ + -B$HOME/src/ITK-wrap-testbed \ + -S. \ + -GNinja \ + -DITK_WRAP_PYTHON:BOOL=ON \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=OFF \ + -DDISABLE_MODULE_TESTS:BOOL=ON \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER:STRING=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \ + -DITK_USE_SYSTEM_CASTXML:BOOL=ON \ + -DITK_USE_SYSTEM_SWIG:BOOL=ON''' +description = "Configure ITK Python wrapping testbed ($HOME/src/ITK-wrap-testbed)" + +[tool.pixi.feature.python.tasks.build-python-local] +cmd = "cmake --build $HOME/src/ITK-wrap-testbed" +description = "Build ITK Python wrapping testbed" +depends-on = ["configure-python-local"] + [tool.pixi.feature.python.tasks.build-python] cmd = "cmake --build build-python" description = "Build ITK Python"