Build main opsqueue binary in Nix using Crane, caching deps

- Make usage of `crane` an implementation detail;
- When building with Nix, people get to customize what `rustToolchain` is
  used, by either setting that in the overlay or directly overriding it
  when using `callPackage`.
- Compile opsqueue_python also with Crane (+ Maturin).
- Document how the Nix Crane<->Maturin setup works
- Bump Rust version in rust-toolchain.toml from 1.85 to 1.92
  This is now also the version used when building with Nix!
- Boyscout rule: Fix warning about unused Cargo.toml lib.include key
- Boyscout rule: Disable integration test timeouts, to test speed of the Nix-based integration suite run on CI

Author: Qqwy

.pre-commit-config.yaml

@@ -25,12 +25,6 @@ repos:
         language: system
         types: [python]
         require_serial: true
-      - id: biome
-        name: Biome formatting
-        entry: biome format --write --files-ignore-unknown=true --no-errors-on-unmatched
-        language: system
-        types: [json]
-        require_serial: true
       - id: rustfmt
         name: rustfmt
         entry: cargo fmt --

biome.json

@@ -47,11 +47,11 @@ test-integration *TEST_ARGS: build-bin build-python
   cd libs/opsqueue_python
   source "./.setup_local_venv.sh"
 
-  timeout 30 pytest --color=yes {{TEST_ARGS}}
+  pytest --color=yes {{TEST_ARGS}}
 
 # Python integration test suite, using artefacts built through Nix. Args are forwarded to pytest
 [group('nix')]
-nix-test-integration *TEST_ARGS:
+nix-test-integration *TEST_ARGS: nix-build
   #!/usr/bin/env bash
   set -euxo pipefail
   nix_build_python_library_dir=$(just nix-build-python)
@@ -61,7 +61,7 @@ nix-test-integration *TEST_ARGS:
   export OPSQUEUE_VIA_NIX=true
   export RUST_LOG="opsqueue=debug"
 
-  timeout 30 pytest --color=yes {{TEST_ARGS}}
+  pytest --color=yes {{TEST_ARGS}}
 
 # Run all linters, fast and slow
 [group('lint')]

justfile

@@ -8,7 +8,7 @@
     # ConsoleSpanExporter,
 )
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource  # type: ignore[attr-defined]
 
 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
 

libs/opsqueue_python/examples/tracing/tracing_consumer.py

@@ -13,7 +13,7 @@
     # ConsoleSpanExporter,
 )
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource  # type: ignore[attr-defined]
 
 from opsqueue.producer import ProducerClient
 
@@ -75,7 +75,7 @@ def added_baggage(
         yield
     finally:
         for attached_token in attached_context_tokens:
-            opentelemetry.context.detach(attached_token)
+            opentelemetry.context.detach(attached_token)  # type: ignore[arg-type]
 
 
 if __name__ == "__main__":

libs/opsqueue_python/examples/tracing/tracing_producer.py

@@ -1,62 +1,92 @@
 {
+  # Builtin
+  pkgs,
   lib,
+
+  # Rust version. (Override this with an overlay if you like)
+  rustToolchain,
+
+  # Native build dependencies:
+  python,
+  maturin,
   buildPythonPackage,
-  rustPlatform,
   perl,
   git,
-  # Python packages:
+
+  # Python package dependencies:
   cbor2,
   opentelemetry-api,
   opentelemetry-exporter-otlp,
   opentelemetry-sdk,
 }:
 let
-  root = ../../.;
-  util = import (root + /nix/util.nix) { inherit lib; };
-in
-buildPythonPackage rec {
-  pname = "opsqueue";
-  version = "0.1.0";
-  pyproject = true;
+  sources = import ../../nix/sources.nix;
+  crane = import sources.crane { pkgs = pkgs; };
+  craneLib = crane.overrideToolchain (pkgs: rustToolchain);
+  extraFileFilter = path: _type: builtins.match ".*(db|sql|py|md)$" path != null;
+  fileFilter = path: type: (extraFileFilter path type) || (craneLib.filterCargoSources path type);
 
-  src = util.fileFilter {
-    name = "opsqueue_python";
-    src = root;
+  src = lib.cleanSourceWith {
+    src = ../../.;
+    name = "opsqueue";
+    filter = fileFilter;
+  };
 
-    # We're copying slightly too much to the Nix store here,
-    # but using the more granular file filter was very error-prone.
-    # This is one thing that could be improved a little in the future.
-    srcGlobalWhitelist = [
-      ".py"
-      ".pyi"
-      "py.typed"
-      ".rs"
-      ".toml"
-      ".lock"
-      ".db"
-      ".md"
-    ];
+  crateName = craneLib.crateNameFromCargoToml { cargoToml = ./Cargo.toml; };
+  pname = crateName.pname;
+  version = crateName.version;
+  commonArgs = {
+    inherit src version pname;
+    strictDeps = true;
+    nativeBuildInputs = [ python ];
+    cargoExtraArgs = "--package opsqueue_python";
   };
+  cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = pname; });
 
-  cargoDeps = rustPlatform.importCargoLock { lockFile = ../../Cargo.lock; };
+  wheelTail = "py3-abi3-linux_x86_64";
+  wheelName = "opsqueue-${version}-${wheelTail}.whl";
 
-  env = {
-    DATABASE_URL = "sqlite://./opsqueue/opsqueue_example_database_schema.db";
-  };
+  crateWheel =
+    (craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; })).overrideAttrs
+      (old: {
+        nativeBuildInputs = old.nativeBuildInputs ++ [ maturin ];
 
-  pythonImportsCheck = [ pname ];
+        # We intentionally _override_ rather than extend the buildPhase
+        # as Maturin itself calls `cargo build`, no need to call it twice.
+        buildPhase = ''
+          cargo --version
+          maturin build --release --offline --target-dir ./target --manifest-path "./libs/opsqueue_python/Cargo.toml"
+        '';
 
-  maturinBuildFlags = [
-    "--manifest-path"
-    "./libs/opsqueue_python/Cargo.toml"
-  ];
+        # We build a single wheel
+        # but by convention its name is based on the precise combination of
+        # Python version + OS version + architecture + ...
+        #
+        # The Nix hash already covers us for uniqueness and compatibility.
+        # So this 'trick' copies it to a predictably named file.
+        #
+        # Just like `buildPhase`, we override rather than extend
+        # because we are only interested in the wheel output of Maturin as a whole.
+        # (which is an archive inside of it containing the `.so` cargo built)
+        installPhase = ''
+          mkdir -p $out
+          for wheel in ./target/wheels/*.whl ; do
+            cp "$wheel" $out/${wheelName}
+          done
+        '';
 
-  nativeBuildInputs = with rustPlatform; [
-    perl
-    git
-    cargoSetupHook
-    maturinBuildHook
-  ];
+        # There are no Rust unit tests in the Python FFI library currently,
+        # so we can skip rebuilding opsqueue_python for tests.
+        doCheck = false;
+      });
+in
+buildPythonPackage {
+  pname = pname;
+  format = "wheel";
+  version = crateName.version;
+  src = "${crateWheel}/${wheelName}";
+  doCheck = false;
+  pythonImportsCheck = [ "opsqueue" ];
 
   propagatedBuildInputs = [
     cbor2

libs/opsqueue_python/opsqueue_python.nix

@@ -52,15 +52,15 @@ test = [
     "pytest==8.3.3",
     "pytest-random-order==1.1.1",
     "pytest-parallel==0.1.1",
-    "pytest-timeout==2.4.0",
+    # "pytest-timeout==2.4.0",
     "py==1.11.0", # Needs to be manually specified because of this issue: https://github.com/kevlened/pytest-parallel/issues/118
 ]
 
 [tool.pytest.ini_options]
 # We ensure tests never rely on global state,
 # by running them in a random order, and in parallel:
-addopts = "--random-order --workers=auto"
-# Individual tests should be very fast. They should never take multiple seconds
-# If after 20sec (accomodating for a toaster-like PC) there is no progress,
-# assume a deadlock
-timeout=20
+addopts = "--random-order --workers=4"
+# # Individual tests should be very fast. They should never take multiple seconds
+# # If after 20sec (accomodating for a toaster-like PC) there is no progress,
+# # assume a deadlock
+# timeout=20

libs/opsqueue_python/pyproject.toml

@@ -428,11 +428,8 @@ pub async fn check_signals_in_background() -> FatalPythonException {
             if let Err(err) = py.check_signals() {
                 // A signal was triggered
                 Some(err)
-            } else if let Some(err) = PyErr::take(py) {
-                // A non-signal Python exception was thrown
-                return Some(err);
             } else {
-                return None;
+                PyErr::take(py)
             }
         });
         if let Some(res) = res {

libs/opsqueue_python/src/common.rs

@@ -418,7 +418,7 @@ impl ConsumerClient {
             }
 
             done_count = done_count.saturating_add(1);
-            if done_count % 50 == 0 {
+            if done_count.is_multiple_of(50) {
                 tracing::info!("Processed {} chunks", done_count);
             }
         }

libs/opsqueue_python/src/consumer.rs

@@ -10,7 +10,8 @@ let
   used_overlays = [
     (import sources.rust-overlay)
     (import ./overlay.nix)
-  ] ++ overlays;
+  ]
+  ++ overlays;
 
   nixpkgsArgs = args // {
     overlays = used_overlays;