[SymForce] Use uv for requirements

For example: aaron-skydio#4

Topic: sf-uv

Author: aaron-skydio

.github/workflows/ci.yml

@@ -121,18 +121,11 @@ jobs:
               libgoogle-glog-dev \
               libeigen3-dev
 
-      # NOTE(aaron): Some packages do not have a version that supports py3.8..py3.12
-      - name: Fix py3.12 versions
-        if: ${{ matrix.python == '3.12' }}
-        run: |
-          sed -i 's|numba==0.58.1|numba~=0.59.0|g' dev_requirements.txt
-          sed -i 's|llvmlite==0.41.1|llvmlite~=0.42.0|g' dev_requirements.txt
-          sed -i 's|numpy==1.24.4|numpy~=1.26.0|g' dev_requirements.txt
-          sed -i 's|scipy==1.10.1|scipy~=1.12.0|g' dev_requirements.txt
-          sed -i 's|pandas==2.0.3|pandas~=2.2.0|g' dev_requirements.txt
-
       - name: Install python dependencies
-        run: pip install -r dev_requirements.txt
+        run: |
+          PY_MINOR_VERSION=$(${{ matrix.python }} -c "import sys; print(sys.version_info.minor)")
+          ${{ matrix.python }} -m pip install pip==24.0 setuptools==69.5.1
+          ${{ matrix.python }} -m pip install -r requirements_dev_py3${PY_MINOR_VERSION}.txt
 
       - name: Run cmake build
         run: |
@@ -144,17 +137,11 @@ jobs:
             -D SYMFORCE_BUILD_BENCHMARKS=ON
           cmake --build build -j $(nproc)
 
-      # - lcmtypes need to be available for tests
-      # - Exact contents of dev_requirements.txt depend on python version. Need to update file to
-      # match current python version to avoid failure of corresponding gen test. symforce needs
-      # to be on the PYTHONPATH to run gen test in this manner.
+      # lcmtypes and symforce need to be available for tests
       - name: Run tests
         run: |
           pip install build/lcmtypes/python2.7
           export PYTHONPATH=$PYTHONPATH:$(pwd)
-          python${{ matrix.python }} test/symforce_requirements_test.py --update
-          echo "Modifications made to requirements:"
-          git diff
           EXIT_CODE=0
           ctest --test-dir build -j $(nproc) || EXIT_CODE=$?
           if [ $EXIT_CODE -ne 0 ]; then

.github/workflows/docs.yml

@@ -23,7 +23,9 @@ jobs:
             pandoc
 
       - name: Install python dependencies
-        run: pip install -r dev_requirements.txt
+        run: |
+          PY_MINOR_VERSION=$(python -c "import sys; print(sys.version_info.minor)")
+          python -m pip install -r requirements_dev_py3${PY_MINOR_VERSION}.txt
 
       - name: Run cmake build
         run: |

.github/workflows/solve_requirements.yml

@@ -0,0 +1,59 @@
+name: Solve Pip Requirements
+
+on:
+  workflow_dispatch:
+    inputs:
+      upgrade:
+        required: true
+        type: boolean
+        description: Run `uv compile` with the `--upgrade` flag
+        default: false
+
+jobs:
+  solve:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        py_version: [8, 9, 10, 11, 12]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.${{ matrix.py_version }}'
+
+    - run: |
+        pip install uv
+        uv pip compile --all-extras --output-file=/tmp/requirements_bootstrap.txt pyproject.toml
+        uv venv
+        source .venv/bin/activate
+        uv pip install -r /tmp/requirements_bootstrap.txt
+        
+    - run: PYTHONPATH=$(pwd) python test/symforce_requirements_test.py --update
+      if: ${{ !github.event.inputs.upgrade }}
+
+    - run: PYTHONPATH=$(pwd) python test/symforce_requirements_test.py --update --piptools_upgrade
+      if: ${{ github.event.inputs.upgrade }}
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: requirements_dev_py3${{ matrix.py_version }}
+        path: requirements_dev_py3${{ matrix.py_version }}.txt
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: solve
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/download-artifact@v4
+      with:
+        merge-multiple: true
+
+    - uses: peter-evans/create-pull-request@v6
+      with:
+        commit-message: Update Requirements
+        title: Update Requirements
+        body: ""

.github/workflows/test_editable_pip_install.yml

@@ -34,18 +34,10 @@ jobs:
             pip==${{ matrix.pip_version }} \
             setuptools==${{ matrix.setuptools_version }}
 
-      # NOTE(aaron): Some packages do not have a version that supports py3.8..py3.12
-      - name: Fix py3.12 versions
-        if: ${{ matrix.os == 'ubuntu-24.04' }}
+      - name: install requirements
         run: |
-          sed -i 's|numba==0.58.1|numba~=0.59.0|g' dev_requirements.txt
-          sed -i 's|llvmlite==0.41.1|llvmlite~=0.42.0|g' dev_requirements.txt
-          sed -i 's|numpy==1.24.4|numpy~=1.26.0|g' dev_requirements.txt
-          sed -i 's|scipy==1.10.1|scipy~=1.12.0|g' dev_requirements.txt
-          sed -i 's|pandas==2.0.3|pandas~=2.2.0|g' dev_requirements.txt
-
-      - name: install dev_requirements.txt
-        run: python -m pip install -r dev_requirements.txt
+          PY_MINOR_VERSION=$(python -c "import sys; print(sys.version_info.minor)")
+          python -m pip install -r requirements_dev_py3${PY_MINOR_VERSION}.txt
 
       - name: editable install
         run: python -m pip install -v -e .

CMakeLists.txt

@@ -5,7 +5,7 @@
 
 # NOTE(aaron): This is the minimum version for policy range support, not sure if we need newer;
 # certainly no newer than 3.15 required.  This will use NEW policies up to CMake 3.25; this should
-# be the maximum tested CMake version, matching dev_requirements.txt
+# be the maximum tested CMake version, matching requirements_dev.txt
 cmake_minimum_required(VERSION 3.19...3.25)
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27)

README.md

@@ -567,7 +567,7 @@ pip install -e .
 
 You should then [verify your installation](#verify-your-installation).
 
-___Note:___ `pip install .` will not install pinned versions of SymForce's dependencies, it'll install any compatible versions.  It also won't install all packages required to run all of the SymForce tests and build all of the targets (e.g. building the docs or running the linters).  If you want all packages required for that, you should `pip install .[dev]` instead (or one of the other groups of extra requirements in our `setup.py`).  If you additionally want pinned versions of our dependencies, which are the exact versions guaranteed by CI to pass all of our tests, you can install them from `pip install -r dev_requirements.txt`.
+___Note:___ `pip install .` will not install pinned versions of SymForce's dependencies, it'll install any compatible versions.  It also won't install all packages required to run all of the SymForce tests and build all of the targets (e.g. building the docs or running the linters).  If you want all packages required for that, you should `pip install .[dev]` instead (or one of the other groups of extra requirements in our `setup.py`).  If you additionally want pinned versions of our dependencies, which are the exact versions guaranteed by CI to pass all of our tests, you can install them from `pip install -r requirements_dev_py3<version>.txt`.
 
 _Note: Editable installs as root with the system python on Ubuntu (and other Debian derivatives) are broken on `setuptools<64.0.0`.  This is a [bug in Debian](https://ffy00.github.io/blog/02-python-debian-and-the-install-locations/), not something in SymForce that we can fix.  If this is your situation, either use a virtual environment, upgrade setuptools to a version `>=64.0.0`, or use a different installation method._
 
@@ -576,11 +576,13 @@ _Note: Editable installs as root with the system python on Ubuntu (and other Deb
 If you'll be modifying the C++ parts of SymForce, you should build with CMake directly instead - this method will not install
 SymForce into your Python environment, so you'll need to add it to your PYTHONPATH separately.
 
-Install python requirements:
+Install dependencies required to build and run SymForce:
 ```bash
-pip install -r dev_requirements.txt
+pip install -r requirements_build.txt
 ```
 
+___Note:___ `requirements_build` contains only packages required to build and run symforce, but not everything recommended to develop symforce, like to run the SymForce tests and linters.  For that, install the full pinned requirements using `pip install -r requirements_dev_py3<version>.txt` for your Python version.
+
 Build SymForce (requires C++14 or later):
 ```bash
 mkdir build