Bump levanter+dolma SHAs, pin lm-eval, scalax SHAs (marin-community#1850)

Bump Levanter from c30de5b to 6cd783c (marin-community#1288), Dolma from fd431d0 to
79ce49d (#2), removing `enable_logprobs` parameter from both
`InferenceEngineConfig` and `Request` classes. The logprobs
functionality was removed in Levanter's inference engine refactor
(marin-community#1277).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>

Author: ryan-williams

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "google-cloud-storage-transfer",
     "jax==0.6.2",
     "haliax>=1.4.dev443",
-    "levanter[serve] @ git+https://github.com/marin-community/levanter.git@c30de5b9ad706da1ddb5e95cf82ed767692f7f7c",
+    "levanter[serve] @ git+https://github.com/marin-community/levanter.git@6cd783c82168919b365c1b2266e4a0bde22ba658",
     "lz4",
     "multiprocess==0.70.16",
     "numpy",
@@ -218,13 +218,13 @@ quality_dedup_consolidate = [
     "cattrs==24.1.3",
     "fsspec>=2025.3.0",
     "ddsketch",
-    "dolma @ git+https://github.com/marin-community/dolma@8b5577fbfb89afc89ea52d20f92e4bc901c5587a",
+    "dolma @ git+https://github.com/marin-community/dolma@79ce49d9535cfc64f4d96781b03ecd0e2c20cf4e",
 ]
 
 tokenize_train = [
     "multiprocess==0.70.16",
     "haliax>=1.4.dev443",
-    "lm-eval@git+https://github.com/stanford-crfm/lm-evaluation-harness.git",
+    "lm-eval@git+https://github.com/stanford-crfm/lm-evaluation-harness@d5e3391f22cde186c827674d5c3ec7c5f4fe0cab",
     "tblib",
     "sentencepiece",
     "tiktoken",
@@ -250,12 +250,12 @@ post_training = [
     "pylatexenc",
     "ipython",
     "datasets",
-    "scalax@git+https://github.com/Sea-Snell/scalax.git",
+    "scalax@git+https://github.com/Sea-Snell/scalax@8cf9ceabe30d4c4274df3c09aae2f66b59fbce3c",
     "swebench ; sys_platform != 'darwin'",
 ]
 
 eval = [
-    "lm-eval[math]@git+https://github.com/stanford-crfm/lm-evaluation-harness.git",
+    "lm-eval[math]@git+https://github.com/stanford-crfm/lm-evaluation-harness@d5e3391f22cde186c827674d5c3ec7c5f4fe0cab",
 ]
 
 

src/marin/rl/rl_job.py

@@ -253,7 +253,6 @@ def to_worker_configs(self) -> tuple[TrainWorkerConfig, RolloutWorkerConfig]:
                     max_seq_len=max_tokens,
                     page_size=128,
                     hbm_utilization=0.5,
-                    enable_logprobs=True,
                 ),
                 port=0,
             )

tests/rl/integration/config.py

@@ -267,7 +267,6 @@ def create_test_inference_server_config(model_config: LlamaConfig, output_dir: s
             page_size=8,
             max_seq_len=64,
             max_queued_tokens=8,
-            enable_logprobs=True,
         ),
         temperature=1.0,
         port=find_open_port(),
@@ -362,7 +361,6 @@ def run_inference_with_engine(
     tokenizer=None,
     max_tokens: int = 64,
     temperature: float = 1.0,
-    enable_logprobs: bool = False,
 ) -> tuple[list[list[int]], list[str]]:
     """Run inference on prompts using InferenceEngine directly."""
     if tokenizer is None:
@@ -374,7 +372,6 @@ def run_inference_with_engine(
         page_size=32,
         max_pages=8 * len(prompts) * max(1, max_tokens // 32),
         compute_dtype=jnp.bfloat16,
-        enable_logprobs=enable_logprobs,
     )
 
     print("Creating inference engine with config:", config)
@@ -402,7 +399,6 @@ def run_inference_with_engine(
             request_id=i,
             decode_params=decode_params,
             n_generations=1,
-            enable_logprobs=enable_logprobs,
         )
         requests.append(request)