refactor: untrack generated python proto, regenerate in build

.github/workflows/build.yml

@@ -180,6 +180,18 @@ jobs:
         with:
           distribution: "temurin"
           java-version: 17
+      - name: Setup Python for genPythonProto
+        # amber/Compile triggers genPythonProto, which shells out to
+        # bin/gen-python-proto.py. The script downloads the pinned protoc
+        # on demand, but still needs python3 and the betterproto plugin
+        # from amber/dev-requirements.txt.
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+      - name: Install Python dependencies for genPythonProto
+        run: |
+          python -m pip install uv
+          uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt
       - name: Create Databases
         # Must run before any sbt compile step: the build's JOOQ source
         # generators connect to texera_db while compiling.
@@ -315,6 +327,8 @@ jobs:
         # mirrors a subset of common deps (e.g. pillow); without this
         # flag a dependabot bump to a version not yet mirrored there
         # fails to resolve even though PyPI has it.
+        # dev-requirements.txt provides the betterproto plugin used by
+        # genPythonProto (wired into amber/Compile in amber/build.sbt).
         run: |
           python -m pip install uv
           if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi
@@ -621,6 +635,8 @@ jobs:
         run: |
           python -m pip install uv
           if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi
+      - name: Generate Python proto bindings
+        run: python3 bin/gen-python-proto.py
       - name: Test with pytest
         run: |
           cd amber && pytest -m "not integration" --cov=src/main/python --cov-report=xml -sv
@@ -664,6 +680,8 @@ jobs:
           if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi
           if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi
           if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi
+      - name: Generate Python proto bindings
+        run: python3 bin/gen-python-proto.py
       - name: Run state-materialization integration tests
         run: |
           cd amber && pytest -sv \

.gitignore

@@ -60,6 +60,9 @@ coverage.xml
 *.model
 *.pkl
 
+# Regenerated by bin/gen-python-proto.py (also via sbt amber/compile).
+amber/src/main/python/proto/
+
 # Ingoring user generated resources
 user-resources/
 

AGENTS.md

@@ -83,6 +83,8 @@ One Python venv shared across worktrees, sibling of the texera checkout:
 ```bash
 python3.12 -m venv ../venv312 && source ../venv312/bin/activate
 pip install -r amber/requirements.txt -r amber/operator-requirements.txt
+# For pytest or sbt-driven Python codegen, also install dev deps:
+pip install -r amber/dev-requirements.txt
 ```
 
 Tests that spawn Python workers need an interpreter path. Edit `python.path`

amber/build.sbt

@@ -179,7 +179,7 @@ libraryDependencies ++= hadoopDependencies
 // protobuf related
 // run the following with sbt to have protobuf codegen
 
-PB.protocVersion := "3.19.4"
+PB.protocVersion := IO.read((ThisBuild / baseDirectory).value / "bin" / "protoc-version.txt").trim
 
 enablePlugins(Fs2Grpc)
 
@@ -200,6 +200,16 @@ libraryDependencies += "com.thesamet.scalapb" %% "scalapb-json4s" % "0.12.0"
 // enable protobuf compilation in Test
 Test / PB.protoSources += PB.externalSourcePath.value
 
+// Wrapper around bin/gen-python-proto.py so `sbt amber/compile` refreshes bindings.
+val genPythonProto = taskKey[Unit]("Generate Python betterproto bindings via bin/gen-python-proto.py.")
+genPythonProto := {
+  val log = streams.value.log
+  val repoRoot = (ThisBuild / baseDirectory).value
+  val exit = scala.sys.process.Process(Seq("python3", "bin/gen-python-proto.py"), repoRoot).!(log)
+  if (exit != 0) sys.error(s"bin/gen-python-proto.py failed with exit code $exit")
+}
+Compile / compile := (Compile / compile).dependsOn(genPythonProto).value
+
 /////////////////////////////////////////////////////////////////////////////
 // Test related
 // https://mvnrepository.com/artifact/org.scalamock/scalamock

amber/dev-requirements.txt

@@ -23,3 +23,8 @@
 # Coverage instrumentation for pytest; emits coverage.xml consumed by
 # Codecov's Phase 1 upload.
 pytest-cov==5.0.0
+
+# protoc plugin invoked by the `genPythonProto` sbt task in
+# amber/build.sbt; runtime needs only the base `betterproto` in
+# requirements.txt.
+betterproto[compiler]==2.0.0b7