CI improvement for checking header regen

.codespellrc

@@ -1,7 +1,11 @@
 # Configuration for codespell (https://github.com/codespell-project/codespell)
 # Shared by the CI workflow and local runs: just run `codespell` from the repo root.
 [codespell]
-skip = ./.git,./build,./third_party,*.svg
+# ./scripts/spec_metadata is a verbatim archive of the specification metadata for
+# the spec release this loader targets. It is not ours to correct -- typos in it
+# have to be fixed upstream in oneapi-src/level-zero-spec and arrive here with the
+# next spec update.
+skip = ./.git,./build,./third_party,*.svg,./scripts/spec_metadata
 # codespell only flags words from its curated misspelling dictionary, but it is not
 # language-aware. Add identifiers/terms that collide with real words here (comma-separated).
 ignore-words-list = te,pevents

.gitattributes

@@ -0,0 +1,7 @@
+# The archived specification metadata is checksummed byte-for-byte --
+# 'input_json_sha256' in scripts/spec_metadata/spec_metadata.json -- so it has to
+# check out identically on every platform. Without this, a Windows checkout with
+# core.autocrlf enabled (the default on the GitHub Windows runners) rewrites its
+# ~138k LFs to CRLF, which changes the file's hash and fails the integrity check
+# before regeneration even starts.
+scripts/spec_metadata/input.json -text

.github/workflows/regen-source.yml

@@ -0,0 +1,152 @@
+name: Regenerate Source
+
+# Most files under source/ are generated by scripts/json2src.py from the Mako
+# templates in scripts/templates/. They are committed, so the build never
+# regenerates them -- which means editing one directly instead of editing its
+# template appears to work right up until the next spec update discards the edit
+# and the build breaks.
+#
+# This job regenerates the sources from the specification metadata archived in
+# scripts/spec_metadata/ and then builds and tests the result. The gate is the
+# build and the tests: if a hand-edit that other code depends on is lost, the
+# regenerated tree fails to compile and this job fails.
+#
+# Textual drift on its own does NOT fail the job. Comment and doc-string wording
+# routinely differs after the spec is edited upstream, and failing on that would
+# train everyone to ignore this check. Drift is reported as inline annotations
+# and uploaded as an artifact instead.
+
+on:
+  push:
+    branches: [ master,release_branch* ]
+  pull_request:
+    branches: [ master,release_branch* ]
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  regen-build-linux:
+    if: github.repository_owner == 'oneapi-src'
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:24.04
+    steps:
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y build-essential cmake git python3 python3-pip
+          # PEP 668 marks the distro interpreter as externally managed.
+          # Floor kept in step with the spec repository's third_party/requirements.txt;
+          # verified to produce byte-identical output from 1.3.3 through 1.3.11.
+          pip3 install --break-system-packages "Mako>=1.3.3"
+      - uses: actions/checkout@v5
+
+      - name: Regenerate sources from archived spec metadata
+        run: python3 scripts/spec_metadata.py regen
+
+      - name: Report drift (advisory)
+        run: |
+          # Git refuses to inspect a checkout owned by another user inside the
+          # container; the workspace is trusted here.
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git diff > regen.diff
+          if [ ! -s regen.diff ]; then
+            echo "No drift: committed sources match a regeneration of the archived metadata."
+            exit 0
+          fi
+          echo "::group::Regeneration diff"
+          cat regen.diff
+          echo "::endgroup::"
+          # Annotate each drifting file. This is advisory only -- the build and
+          # test steps below decide whether the job passes.
+          git diff --name-only | while read -r f; do
+            echo "::warning file=${f},line=1::Regenerating from the archived spec metadata changes this file. If the change here was hand-written, move it into the matching scripts/templates/*.mako template or it will be lost on the next spec update."
+          done
+
+      - name: Upload drift report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: regen-diff-linux
+          path: regen.diff
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Build the regenerated tree
+        run: |
+          cmake -B build \
+            -D CMAKE_BUILD_TYPE=Release \
+            -D BUILD_L0_LOADER_TESTS=1 \
+            -D INSTALL_NULL_DRIVER=1 \
+            .
+          if ! cmake --build build --parallel $(nproc); then
+            echo "::error::The regenerated tree does not compile."
+            echo ""
+            echo "This almost always means a change was made directly to a generated"
+            echo "file under source/ instead of to its template in scripts/templates/."
+            echo "Regeneration discarded that change, so something that depended on it"
+            echo "no longer builds."
+            echo ""
+            echo "Fix: move the change into the matching scripts/templates/*.mako"
+            echo "template, then run 'python3 scripts/spec_metadata.py regen' and commit"
+            echo "the regenerated files."
+            exit 1
+          fi
+
+      - name: Test the regenerated tree
+        working-directory: build
+        env:
+          ZEL_LIBRARY_PATH: '${{ github.workspace }}/build/lib'
+          LD_LIBRARY_PATH: '${{ github.workspace }}/build/lib'
+        run: ctest --output-on-failure
+
+  regen-build-windows:
+    if: github.repository_owner == 'oneapi-src'
+    runs-on: [windows-latest]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        # See the Linux job for why this tracks the spec's requirements.txt.
+        run: pip install "Mako>=1.3.3"
+      - name: Setup Windows build environment
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: Regenerate sources from archived spec metadata
+        run: python3 scripts/spec_metadata.py regen
+
+      - name: Report drift (advisory)
+        shell: bash
+        run: |
+          git diff > regen.diff
+          if [ ! -s regen.diff ]; then
+            echo "No drift: committed sources match a regeneration of the archived metadata."
+            exit 0
+          fi
+          echo "::group::Regeneration diff"
+          cat regen.diff
+          echo "::endgroup::"
+
+      - name: Build the regenerated tree
+        shell: bash
+        run: |
+          cmake -B build -G "Ninja" -D CMAKE_BUILD_TYPE=Release -D BUILD_L0_LOADER_TESTS=1 .
+          if ! cmake --build build; then
+            echo "::error::The regenerated tree does not compile."
+            echo "A change was likely made directly to a generated file under source/"
+            echo "instead of to its template in scripts/templates/, and regeneration"
+            echo "discarded it. Move the change into the template, run"
+            echo "'python3 scripts/spec_metadata.py regen', and commit the result."
+            exit 1
+          fi
+
+      - name: Test the regenerated tree
+        working-directory: build
+        env:
+          ZEL_LIBRARY_PATH: '${{ github.workspace }}/build/bin'
+        run: ctest --output-on-failure

.github/workflows/update-spec.yml

@@ -10,11 +10,15 @@ on:
         type: string
         default: oneapi-src/level-zero-spec
       ref:
-        description: Ref to checkout from the spec repository
+        description: >-
+          Ref to checkout from the spec repository. Resolved to the most recent
+          release tag reachable from it, since only official releases are imported.
         type: string
         default: master
       version:
-        description: Version of the spec to generate code for
+        description: >-
+          API version to generate code for, e.g. 1.17. Leave blank to derive it
+          from the most recent release tag in the spec repository.
         type: string
       branch:
         description: Branch to (force) push generated code to
@@ -33,7 +37,9 @@ jobs:
       with:
         python-version: '3.x'
     - name: Install dependencies
-      run: pip install Mako==1.1.0 PyYAML==5.2
+      # Kept in step with the spec repository's third_party/requirements.txt, since
+      # this job runs that repository's generators.
+      run: pip install "Mako>=1.3.3" "PyYAML>=6.0.1"
     - uses: actions/checkout@v5
       with:
         clean: true
@@ -56,28 +62,96 @@ jobs:
         fetch-depth: 0
         path: spec
         ref: ${{ github.event_name == 'pull_request' && 'master' || inputs.ref }}
+    - name: Resolve the spec release to import
+      run: | #bash
+        # This job imports *official spec releases*, so the checkout is pinned to a
+        # release tag rather than left on a moving branch. Between releases, master
+        # carries post-release commits: generating from it would take that content
+        # while 'git describe' still reports the previous tag, and the metadata
+        # archived below would claim to be a release it is not. That is not
+        # hypothetical -- spec PR #505 landed after v1.18.0 was cut, which is
+        # exactly how the loader's generated sources came to drift.
+        #
+        # --match restricts this to release tags, and --abbrev=0 takes the most
+        # recent one *reachable from the checkout* rather than the highest tag in
+        # the clone. Those differ once a newer release exists, and importing
+        # v1.17.24 must stay on 1.17 even when v1.18.0 has been cut. This is the
+        # same rule as the spec's own run.py:detect_version_git_major_minor().
+        CHECKED_OUT=$(git -C spec rev-parse --short HEAD)
+        SPEC_TAG=$(git -C spec describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*' 2>/dev/null || true)
+        if [ -z "${SPEC_TAG}" ]; then
+            echo "::error::No release tag is reachable from the requested spec ref (${CHECKED_OUT}). This job imports official spec releases only."
+            exit 1
+        fi
+        git -C spec checkout --quiet --detach "${SPEC_TAG}"
+        RESOLVED=$(git -C spec rev-parse --short HEAD)
+        if [ "${RESOLVED}" != "${CHECKED_OUT}" ]; then
+            echo "::notice::Spec checkout moved from ${CHECKED_OUT} to release tag ${SPEC_TAG} (${RESOLVED}); commits made after the tag are not imported."
+        fi
+        echo "Importing spec release ${SPEC_TAG} (${RESOLVED})"
+
+        # The API version is a max-version filter over the spec metadata, not a
+        # label: set it below the release being imported and every function added
+        # after that version is silently dropped -- the loader still compiles, it
+        # just stops exporting them. Derive it from the resolved tag rather than
+        # writing it down here, where it would go stale.
+        REQUESTED_VER="${{ inputs.version }}"
+        if [ -n "${REQUESTED_VER}" ]; then
+            VER="${REQUESTED_VER}"
+            echo "Using requested API version: ${VER}"
+        else
+            VER=$(echo "${SPEC_TAG}" | sed -n 's/^v\([0-9]\+\.[0-9]\+\).*/\1/p')
+            if [ -z "${VER}" ]; then
+                echo "::error::Could not parse a MAJOR.MINOR version from tag '${SPEC_TAG}'."
+                exit 1
+            fi
+            echo "Detected API version: ${VER} (from ${SPEC_TAG})"
+        fi
+        echo "VER=${VER}" >> $GITHUB_ENV
+        echo "SPEC_TAG=${SPEC_TAG}" >> $GITHUB_ENV
     - name: Apply latest spec commit to develop branch
       run: | #bash
         export LANG="C.UTF-8"
         cd spec/scripts
         echo "::group::Generate spec/scripts/input.json"
-        python3 ./run.py --debug '--!html' '--!rst' '--!build' --ver ${{ github.event_name == 'pull_request' && '1.10' || inputs.version }}
+        python3 ./run.py --debug '--!html' '--!rst' '--!build' --ver "${VER}"
         echo "::endgroup::"
         cd ..
         echo "::group::Copy generated header files"
         jq -r '.[] | select(.|test("^../include"))' scripts/generated.json | cut -c 4- | sort | xargs -I{} cp -v ./{} ../{}
         echo "::endgroup::"
         cd ..
+        # Archive the metadata alongside the code it produced. The Regenerate
+        # Source workflow replays it on every pull request, so it has to describe
+        # the same spec release as the headers being committed here.
+        echo "::group::Archive spec metadata"
+        python3 ./scripts/spec_metadata.py refresh \
+            --spec-repo spec \
+            --ref "${SPEC_TAG}" \
+            --ver "${VER}" \
+            --input-json spec/scripts/input.json
+        echo "::endgroup::"
+        # Generate through the same entry point the Regenerate Source workflow
+        # uses, rather than calling json2src.py directly. A second generation path
+        # here would only have to differ in how it invokes the generator for every
+        # post-bump pull request to report drift against the very commit that
+        # produced it. Going through 'regen' makes the archive and the committed
+        # sources consistent by construction.
         echo "::group::Generate source files"
-        ./scripts/json2src.py < spec/scripts/input.json --ver ${{ github.event_name == 'pull_request' && '1.10' || inputs.version }} .
+        python3 ./scripts/spec_metadata.py regen
         echo "::endgroup::"
-        if (($(git diff | tee >(wc -l) >&2) == 0)); then
+        # Summarise the change without dumping the multi-megabyte metadata blob
+        # into the log.
+        git diff --stat -- . ':(exclude)scripts/spec_metadata' >&2
+        if git diff --quiet; then
             echo "::warning::No changes were made to files"
         else
             git config user.email "sys-lzdev@intel.com"
             git config user.name "sys-lzdev"
             spec_ver=$(sed -n 's/^.*version v.*-r\([0-9]*\.[0-9]*\.[0-9]*\)/\1/p' include/ze_api.h)
             git add -u
+            # -u alone would miss the archive the first time it appears.
+            git add -A scripts/spec_metadata
             git commit -m "Update to spec ${spec_ver}"
         fi
     - name: Push changes to develop branch

CMakeLists.txt

@@ -225,6 +225,30 @@ if(NOT BUILD_INSTALLER AND EXISTS "/etc/debian_version")
     set(CANONICAL_SDK_COMPONENT "libze-dev")
 endif()
 
+if (MSVC)
+    set (PYTHON_EXECUTABLE "python")
+else()
+    set (PYTHON_EXECUTABLE "python3")
+endif()
+
+# Most of the files under source/ are generated by scripts/json2src.py from the
+# Mako templates in scripts/templates/, using the specification metadata archived
+# in scripts/spec_metadata/. They are committed, so a normal build never
+# regenerates them and needs no Python at all.
+#
+# This target is a convenience wrapper around the real entry point:
+#   python3 scripts/spec_metadata.py regen
+# It rewrites committed files in the source tree, so it only ever runs when named
+# explicitly -- deliberately not wired into the default build or into configure,
+# where a cached option would silently rewrite the tree on every reconfigure.
+#   cmake --build <dir> --target regenerate-source
+add_custom_target(regenerate-source
+    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/spec_metadata.py regen
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Regenerating loader source from archived spec metadata"
+    VERBATIM
+)
+
 add_subdirectory(source)
 add_subdirectory(samples)
 
@@ -290,11 +314,7 @@ if(CANONICAL_SDK_COMPONENT)
     )
 endif()
 
-if (MSVC)
-    set (PYTHON_EXECUTABLE "python")
-else()
-    set (PYTHON_EXECUTABLE "python3")
-endif()
+# PYTHON_EXECUTABLE is set above, next to the source regeneration options.
 set(PRODUCT_GUID_FILE "${CMAKE_CURRENT_SOURCE_DIR}/PRODUCT_GUID.txt")
 if(EXISTS "${PRODUCT_GUID_FILE}")
     file(STRINGS "${PRODUCT_GUID_FILE}" SAVED_PRODUCT_GUID)