diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 411c3e3a73e..7936743277b 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -58,6 +58,10 @@ jobs: ) ) + env: + MANIFEST_PATH: ci/nightly/pixi.toml + LOCK_PATH: ci/nightly/pixi.lock + outputs: cache-key: ${{ steps.pixi-lock.outputs.cache-key }} pixi-version: ${{ steps.pixi-lock.outputs.pixi-version }} @@ -65,12 +69,17 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: Parcels-code/pixi-lock/create-and-cache@38495788b79a5ff26009aecc15daa9a8310b8832 # v0.1.0 + - uses: Parcels-code/pixi-lock/create-and-cache@1fab4c17af8243f6ca1321bc941cf2325f46c05d # v0.1.1 id: pixi-lock + with: + manifest-path: ${{ env.MANIFEST_PATH }} + hash-files: | + ${{ env.MANIFEST_PATH }} + pyproject.toml - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pixi-lock - path: pixi.lock + path: ${{ env.LOCK_PATH }} upstream-dev: name: upstream-dev @@ -89,6 +98,7 @@ jobs: env: ENV: ${{ matrix.pixi-env }} + MANIFEST_PATH: ci/nightly/pixi.toml outputs: log-file: ${{ steps.determine-log-path.outputs.log-file }} @@ -102,31 +112,35 @@ jobs: id: determine-log-path run: | echo "log-file=output-${ENV}-log.jsonl" >> $GITHUB_OUTPUT; - cat $GITHUB_OUTPUT - name: Restore cached pixi lockfile - uses: Parcels-code/pixi-lock/restore@38495788b79a5ff26009aecc15daa9a8310b8832 # v0.1.0 + uses: Parcels-code/pixi-lock/restore@1fab4c17af8243f6ca1321bc941cf2325f46c05d # v0.1.1 with: + manifest-path: ${{ env.MANIFEST_PATH }} cache-key: ${{ needs.cache-pixi-lock.outputs.cache-key }} - uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0 with: pixi-version: ${{ needs.cache-pixi-lock.outputs.pixi-version }} cache: true environments: ${{ matrix.pixi-env }} + manifest-path: ${{ env.MANIFEST_PATH }} cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - name: Version info run: | - pixi run -e $ENV -- python xarray/util/print_versions.py + pixi run --manifest-path "$MANIFEST_PATH" -e $ENV -- python xarray/util/print_versions.py - name: Import xarray run: | - pixi run -e $ENV -- python -c 'import xarray' + pixi run --manifest-path "$MANIFEST_PATH" -e $ENV -- python -c 'import xarray' - name: Run Tests if: success() id: status env: LOG_PATH: ${{ steps.determine-log-path.outputs.log-file }} run: | - pixi run -e $ENV -- python -m pytest --timeout=60 -rf -nauto \ - --report-log "$LOG_PATH" + pixi run \ + --manifest-path "$MANIFEST_PATH" \ + -e $ENV \ + -- \ + python -m pytest --timeout=60 -rf -nauto --report-log "$LOG_PATH" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: failure() with: @@ -174,6 +188,7 @@ jobs: env: ENV: ${{ matrix.pixi-env }} + MANIFEST_PATH: ci/nightly/pixi.toml steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -182,26 +197,28 @@ jobs: persist-credentials: false - name: Restore cached pixi lockfile - uses: Parcels-code/pixi-lock/restore@38495788b79a5ff26009aecc15daa9a8310b8832 # v0.1.0 + uses: Parcels-code/pixi-lock/restore@1fab4c17af8243f6ca1321bc941cf2325f46c05d # v0.1.1 with: + manifest-path: ${{ env.MANIFEST_PATH }} cache-key: ${{ needs.cache-pixi-lock.outputs.cache-key }} - uses: prefix-dev/setup-pixi@a09b6247153796b190642a2b53fac4241043cf6f # v0.10.0 with: pixi-version: ${{ needs.cache-pixi-lock.outputs.pixi-version }} cache: true environments: ${{ matrix.pixi-env }} + manifest-path: ${{ env.MANIFEST_PATH }} cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - name: set environment variables run: | echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - echo "PYTHON_VERSION=$(pixi run -e $ENV -- python --version | cut -d' ' -f2 | cut -d. -f1,2)" >> $GITHUB_ENV + echo "PYTHON_VERSION=$(pixi run --manifest-path "$MANIFEST_PATH" -e $ENV -- python --version | cut -d' ' -f2 | cut -d. -f1,2)" >> $GITHUB_ENV - name: Version info run: | - pixi run -e $ENV -- python xarray/util/print_versions.py + pixi run --manifest-path "$MANIFEST_PATH" -e $ENV -- python xarray/util/print_versions.py - name: Run mypy run: | - pixi run -e $ENV -- python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report + pixi run --manifest-path "$MANIFEST_PATH" -e $ENV -- python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report - name: Upload mypy coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f2ea760edc9..fa081448061 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,6 +14,7 @@ build: - asdf plugin add pixi - asdf install pixi latest - asdf global pixi latest + - pixi lock --no-install post_checkout: - (git --no-pager log --pretty="tformat:%s" -1 | grep -vqF "[skip-rtd]") || exit 183 - git fetch --unshallow || true diff --git a/ci/nightly/pixi.toml b/ci/nightly/pixi.toml new file mode 100644 index 00000000000..0f4f640c95e --- /dev/null +++ b/ci/nightly/pixi.toml @@ -0,0 +1,146 @@ +[workspace] +preview = ["pixi-build"] +channels = ["conda-forge", "nodefaults"] +platforms = ["linux-64"] +requires-pixi = ">=0.63.2,!=0.68.0" + +[feature.test.dependencies] +pytest = "!=9.1.0" +pytest-asyncio = "*" +pytest-cov = "*" +pytest-env = "*" +pytest-mypy-plugins = ">=4.0.0" +pytest-reportlog = "*" +pytest-timeout = "*" +pytest-xdist = "*" +pytz = "*" +hypothesis = "*" +coveralls = "*" + +[feature.test.tasks] +test = { cmd = "pytest", description = "Run the test suite with pytest." } + +[feature.py314.dependencies] +python = "3.14.*" + +# TODO: Remove `target.unix` restriction once pandas nightly has win-64 wheels again. +# Without this, `pixi lock` fails because it can't solve the nightly feature for win-64, +# which breaks RTD builds (RTD has no lock file cache, unlike GitHub Actions CI). +[feature.nightly.target.unix.dependencies] +python = "*" + +[feature.nightly.pypi-options.dependency-overrides] +numpy = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } +scipy = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } +matplotlib = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } +pandas = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } +pyarrow = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } + +dask = { git = "https://github.com/dask/dask" } +distributed = { git = "https://github.com/dask/distributed" } +zarr = { git = "https://github.com/zarr-developers/zarr-python" } +# numcodecs = { git = "https://github.com/zarr-developers/numcodecs" } #? compilation fails currently, see #11437 +cftime = { git = "https://github.com/Unidata/cftime" } +# packaging = { git = "https://github.com/pypa/packaging"} #? Pixi warns if this is enabled +pint = { git = "https://github.com/hgrecco/pint" } +bottleneck = { git = "https://github.com/pydata/bottleneck" } +fsspec = { git = "https://github.com/intake/filesystem_spec" } +nc-time-axis = { git = "https://github.com/SciTools/nc-time-axis" } +flox = { git = "https://github.com/xarray-contrib/flox" } +h5netcdf = { git = "https://github.com/h5netcdf/h5netcdf" } +opt_einsum = { git = "https://github.com/dgasmith/opt_einsum" } +# sparse = { git = "https://github.com/pydata/sparse"} + +[feature.nightly.target.unix.pypi-dependencies] +xarray = { path = "../..", editable = true } +numpy = "*" +pandas = "*" +matplotlib = "*" +scipy = "*" +pyarrow = "*" + +dask = "*" +distributed = "*" +zarr = "*" +numcodecs = "*" +cftime = "*" +packaging = "*" +pint = "*" +bottleneck = "*" +fsspec = "*" +nc-time-axis = "*" +flox = "*" +# h5netcdf = "*" +# h5py = "*" +opt_einsum = "*" +netcdf4 = "*" +scitools-iris = "*" +pydap = "*" +cartopy = "*" +seaborn = "*" + +[feature.typing.dependencies] +mypy = "==1.19.1" +pyright = "*" +hypothesis = "*" +lxml = "*" +types-colorama = "*" +types-docutils = "*" +types-decorator = "*" +types-networkx = "*" +types-openpyxl = "*" +types-psutil = "*" +types-Pygments = "*" +types-python-dateutil = "*" +types-pytz = "*" +types-PyYAML = "*" +types-requests = "*" +types-setuptools = "*" +typing_extensions = "*" +pip = "*" + +[feature.typing.pypi-dependencies] +types-defusedxml = "*" +types-pexpect = "*" + +# Stubs sourced from PyPI rather than conda, used only by the `mypy-upstream` +# environment. That env pairs the `nightly` feature (which overrides numpy/scipy +# to PyPI nightly wheels) with type stubs; conda stubs would pull conda +# numpy/scipy that pixi cannot reconcile with those PyPI overrides, so they must +# come from PyPI here. +# +# NOTE: these stubs cap numpy below the nightly wheel (scipy-stubs via optype -> +# numpy-typing-compat needs numpy<2.5; pandas-stubs needs numpy<=2.3.5), so numpy +# in `mypy-upstream` resolves to the latest release. We accept that: dropping the +# stubs to free up numpy would leave pandas/scipy untyped and flood mypy with +# import-untyped errors. The upstream *test* job (`test-nightly`) is kept separate +# precisely so it still gets the genuine nightly numpy. +[feature.typing-stubs-nightly.pypi-dependencies] +pandas-stubs = "<=2.3.3.251219" +scipy-stubs = ">=1.17.1.2" + +[feature.typing.tasks] +mypy = { cmd = "mypy --install-types --non-interactive --cobertura-xml-report mypy_report", description = "Run mypy type checking and generate a Cobertura XML report." } + +[environments] +test-nightly = { features = [ + "py314", + "nightly", + "test", + # Deliberately no "typing": type stubs cap numpy<2.5 and would displace the + # nightly numpy wheel this env's pytest job needs. Type checking against the + # dev stack lives in the separate `mypy-upstream` environment below. +], no-default-feature = true } + +# Type-checking against the development stack (used by the `mypy-upstream-dev` CI +# job). Same as `test-nightly` plus the typing toolchain and PyPI-sourced stubs. +# numpy resolves to the latest release here (the stubs cap it below the nightly +# wheel); every other dependency still comes from the nightly wheels / git, so +# this catches typing breakage from upstream development versions. +mypy-upstream = { features = [ + "py314", + "nightly", + "test", + "typing", + "typing-stubs-nightly", +], no-default-feature = true } diff --git a/pixi.toml b/pixi.toml index 0ae5b4341c2..52dda513e46 100644 --- a/pixi.toml +++ b/pixi.toml @@ -143,62 +143,6 @@ toolz = "1.0.*" zarr = "3.0.*" pyarrow = "*" # required by dask.dataframe -# TODO: Remove `target.unix` restriction once pandas nightly has win-64 wheels again. -# Without this, `pixi lock` fails because it can't solve the nightly feature for win-64, -# which breaks RTD builds (RTD has no lock file cache, unlike GitHub Actions CI). -[feature.nightly.target.unix.dependencies] -python = "*" - -[feature.nightly.pypi-options.dependency-overrides] -numpy = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } -scipy = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } -matplotlib = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } -pandas = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } -pyarrow = { version = "*", index = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" } - -dask = { git = "https://github.com/dask/dask" } -distributed = { git = "https://github.com/dask/distributed" } -zarr = { git = "https://github.com/zarr-developers/zarr-python" } -# numcodecs = { git = "https://github.com/zarr-developers/numcodecs" } #? compilation fails currently, see #11437 -cftime = { git = "https://github.com/Unidata/cftime" } -# packaging = { git = "https://github.com/pypa/packaging"} #? Pixi warns if this is enabled -pint = { git = "https://github.com/hgrecco/pint" } -bottleneck = { git = "https://github.com/pydata/bottleneck" } -fsspec = { git = "https://github.com/intake/filesystem_spec" } -nc-time-axis = { git = "https://github.com/SciTools/nc-time-axis" } -flox = { git = "https://github.com/xarray-contrib/flox" } -h5netcdf = { git = "https://github.com/h5netcdf/h5netcdf" } -opt_einsum = { git = "https://github.com/dgasmith/opt_einsum" } -# sparse = { git = "https://github.com/pydata/sparse"} - -[feature.nightly.target.unix.pypi-dependencies] -xarray = { path = ".", editable = true } - -numpy = "*" -pandas = "*" -matplotlib = "*" -scipy = "*" -pyarrow = "*" - -dask = "*" -distributed = "*" -zarr = "*" -numcodecs = "*" -cftime = "*" -packaging = "*" -pint = "*" -bottleneck = "*" -fsspec = "*" -nc-time-axis = "*" -flox = "*" -# h5netcdf = "*" -# h5py = "*" -opt_einsum = "*" -netcdf4 = "*" -scitools-iris = "*" -pydap = "*" -cartopy = "*" -seaborn = "*" [feature.test.dependencies] pytest = "!=9.1.0" @@ -300,22 +244,6 @@ types-pexpect = "*" pandas-stubs = "<=2.3.3.251219" scipy-stubs = ">=1.17.1.2" -# Stubs sourced from PyPI rather than conda, used only by the `mypy-upstream` -# environment. That env pairs the `nightly` feature (which overrides numpy/scipy -# to PyPI nightly wheels) with type stubs; conda stubs would pull conda -# numpy/scipy that pixi cannot reconcile with those PyPI overrides, so they must -# come from PyPI here. -# -# NOTE: these stubs cap numpy below the nightly wheel (scipy-stubs via optype -> -# numpy-typing-compat needs numpy<2.5; pandas-stubs needs numpy<=2.3.5), so numpy -# in `mypy-upstream` resolves to the latest release. We accept that: dropping the -# stubs to free up numpy would leave pandas/scipy untyped and flood mypy with -# import-untyped errors. The upstream *test* job (`test-nightly`) is kept separate -# precisely so it still gets the genuine nightly numpy. -[feature.typing-stubs-nightly.pypi-dependencies] -pandas-stubs = "<=2.3.3.251219" -scipy-stubs = ">=1.17.1.2" - [feature.typing.tasks] mypy = { cmd = "mypy --install-types --non-interactive --cobertura-xml-report mypy_report", description = "Run mypy type checking and generate a Cobertura XML report." } @@ -393,6 +321,9 @@ numpydoc-lint = { cmd = "python ci/numpydoc-public-api.py" } [feature.nodask.constraints] dask-core = "<0.0a0" +[feature.nightly.dependencies] +python = "*" + [environments] # Testing # test-just-xarray = { features = ["test"] } # https://github.com/pydata/xarray/pull/10888/files#r2511336147 @@ -435,28 +366,6 @@ test-py314 = { features = [ "viz", "extras", ] } -test-nightly = { features = [ - "py313", - "nightly", - "test", - # Deliberately no "typing": type stubs cap numpy<2.5 and would displace the - # nightly numpy wheel this env's pytest job needs. Type checking against the - # dev stack lives in the separate `mypy-upstream` environment below. -], no-default-feature = true } - -# Type-checking against the development stack (used by the `mypy-upstream-dev` CI -# job). Same as `test-nightly` plus the typing toolchain and PyPI-sourced stubs. -# numpy resolves to the latest release here (the stubs cap it below the nightly -# wheel); every other dependency still comes from the nightly wheels / git, so -# this catches typing breakage from upstream development versions. -mypy-upstream = { features = [ - "py313", - "nightly", - "test", - "typing", - "typing-stubs-nightly", -], no-default-feature = true } - test-py311 = { features = [ "py311", @@ -535,3 +444,4 @@ default = { features = [ policy = { features = ["policy"], no-default-feature = true } numpydoc-lint = { features = ["numpydoc-lint"] } tools = { features = ["tools"], no-default-feature = true } +test-nightly = { features = ["nightly"], no-default-feature = true }